From f3018eaeb1e03c03f0099b77a1ab1ad3d12217ef Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Tue, 9 Jun 2026 14:59:24 +0000 Subject: [PATCH 1/3] agent: Scope tools by app name using @AppName prefix Implement a temporary bootstrapping method to scope the agent's available tools to a specific app by prefixing the user message with `@AppName`. Add parsing logic in `AgentOrchestrator.kt` to extract `@AppName` and match it against installed apps. Sort installed apps by descending label length to resolve ambiguities (e.g., matching "@Google Maps" before "@Google"). Filter tools passed to the LLM to only include those from the matched app package. Strip the `@AppName ` prefix from the prompt sent to the LLM. Add unit test `observeAndProcessMessages scopes tools by app name` in `AgentOrchestratorTest.kt` to verify filtering and prefix stripping. This is a temporary solution for b/508130322 and will be replaced by UI-provided metadata in b/521319810. BUG=b/508130322 Change-Id: If7d06388f5023b26f952074d8ab5de62eb23ba01 --- .../agent/domain/AgentOrchestrator.kt | 134 +++++-- .../agent/domain/AgentOrchestratorTest.kt | 337 ++++++++++++++---- 2 files changed, 362 insertions(+), 109 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index cee4777..2d79804 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -27,6 +27,7 @@ import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFuncti import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -64,6 +65,7 @@ class AgentOrchestrator private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, + private val getInstalledAppsUseCase: GetInstalledAppsUseCase, ) { private val _status = MutableStateFlow(AgentStatus.Idle) @@ -95,7 +97,6 @@ class AgentOrchestrator try { val provider = thread.llmModel.providerName - val modelName = thread.llmModel.modelName val apiKey = getApiKey(provider) if (apiKey == null) { completeMessageWithError( @@ -108,39 +109,20 @@ class AgentOrchestrator } val disconnectedApps = settingsRepository.disconnectedApps.first() - val tools = - getAppFunctionsUseCase().first().values.flatten().filter { metadata -> - metadata.isEnabled && metadata.packageName !in disconnectedApps - } - var previousInteractionId = thread.latestInteractionId - val currentInput = message.textContent - var currentToolOutputs = emptyList() - var continueLoop = true - - val llmProvider = llmProviderFactory.getProvider(provider) - - while (continueLoop) { - val llmInput = prepareLlmInput(currentToolOutputs, currentInput) - - currentToolOutputs = emptyList() - val response = - llmProvider.generateResponse( - previousInteractionId = previousInteractionId, - input = llmInput, - tools = tools, - apiKey = apiKey, - modelName = modelName, - ) - when (val handleResult = handleLlmResponse(response, message, tools)) { - is HandleResult.Continue -> { - currentToolOutputs = handleResult.toolOutputs - previousInteractionId = handleResult.interactionId - } - is HandleResult.Stop -> { - continueLoop = false - } - } - } + val allTools = getAppFunctionsUseCase().first().values.flatten() + + val (targetPackageName, queryText) = parseTargetPackage(message.textContent) + + val tools = filterTools(allTools, disconnectedApps, targetPackageName) + + runInteractionLoop( + message = message, + thread = thread, + apiKey = apiKey, + tools = tools, + initialInput = queryText, + ) + _status.value = AgentStatus.Idle } catch (e: Exception) { Log.e("AgentOrchestrator", "Error processing message", e) @@ -153,6 +135,85 @@ class AgentOrchestrator } } + // TODO: b/521319810 - This text-based parsing is a temporary bootstrapping method implemented in b/508130322. + // In b/521319810, the UI autocomplete menu will return a unique package name + // and attach it directly to the MessageEntity metadata. + // Once that is implemented, this entire `@` parsing block and `getInstalledAppsUseCase` lookup + // should be replaced by reading `targetPackageName` directly from the message metadata. + private suspend fun parseTargetPackage(content: String): Pair { + if (!content.startsWith("@")) { + return Pair(null, content) + } + + val installedApps = getInstalledAppsUseCase() + val sortedApps = installedApps.sortedByDescending { it.label.length } + + for (app in sortedApps) { + val prefix = "@${app.label}" + if (content.equals(prefix, ignoreCase = true)) { + return Pair(app.packageName, "") + } else if (content.startsWith("$prefix ", ignoreCase = true)) { + return Pair(app.packageName, content.substring(prefix.length + 1).trim()) + } + } + + return Pair(null, content) + } + + private fun filterTools( + allTools: List, + disconnectedApps: Set, + targetPackageName: String?, + ): List { + return allTools.filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + } + + private suspend fun runInteractionLoop( + message: MessageEntity, + thread: ThreadEntity, + apiKey: String, + tools: List, + initialInput: String, + ) { + val provider = thread.llmModel.providerName + val modelName = thread.llmModel.modelName + val llmProvider = llmProviderFactory.getProvider(provider) + + var previousInteractionId = thread.latestInteractionId + var currentToolOutputs = emptyList() + var continueLoop = true + var currentInput = initialInput + + while (continueLoop) { + val llmInput = prepareLlmInput(currentToolOutputs, currentInput) + + currentToolOutputs = emptyList() + val response = + llmProvider.generateResponse( + previousInteractionId = previousInteractionId, + input = llmInput, + tools = tools, + apiKey = apiKey, + modelName = modelName, + ) + + when (val handleResult = handleLlmResponse(response, message, tools)) { + is HandleResult.Continue -> { + currentToolOutputs = handleResult.toolOutputs + previousInteractionId = handleResult.interactionId + } + + is HandleResult.Stop -> { + continueLoop = false + } + } + } + } + private suspend fun getApiKey(provider: LlmProviderName): String? { return when (provider) { LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() @@ -222,6 +283,7 @@ class AgentOrchestrator } HandleResult.Continue(toolResult.toolOutputs, response.interactionId) } + is ExecuteToolCallsResult.PendingIntentAction -> { savePendingIntentUseCase( toolResult.pendingIntentId, @@ -236,6 +298,7 @@ class AgentOrchestrator ) HandleResult.Stop } + is ExecuteToolCallsResult.Error -> { HandleResult.Stop } @@ -252,6 +315,7 @@ class AgentOrchestrator HandleResult.Stop } } + is LlmResponse.Error -> { Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") completeMessageWithError(message.messageId, message.threadId, response.errorMessage) @@ -321,6 +385,7 @@ class AgentOrchestrator ), ) } + is ExecuteAppFunctionResult.PendingIntentAction -> { val pendingIntentId = java.util.UUID.randomUUID().toString() return ExecuteToolCallsResult.PendingIntentAction( @@ -328,6 +393,7 @@ class AgentOrchestrator executionResult.pendingIntent, ) } + is ExecuteAppFunctionResult.Error -> throw IllegalStateException( "Tool execution failed for ${toolCall.functionId}: ${executionResult.exception.message}", diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 724192a..0543f27 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -15,6 +15,8 @@ */ package com.example.appfunctions.agent.domain +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository @@ -22,9 +24,11 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity +import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -35,6 +39,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase import io.mockk.coEvery import io.mockk.coVerify +import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.delay import kotlinx.coroutines.flow.flow @@ -60,6 +65,8 @@ class AgentOrchestratorTest { private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) + private val getInstalledAppsUseCase: GetInstalledAppsUseCase = mockk() + private lateinit var agentOrchestrator: AgentOrchestrator @Before @@ -77,6 +84,7 @@ class AgentOrchestratorTest { convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, savePendingIntentUseCase = savePendingIntentUseCase, + getInstalledAppsUseCase = getInstalledAppsUseCase, ) } @@ -84,30 +92,11 @@ class AgentOrchestratorTest { fun `observeAndProcessMessages fails when API key is missing`() = runTest { val threadId = "thread_1" - val message = - MessageEntity( - messageId = "msg_1", - threadId = threadId, - role = MessageRole.USER, - textContent = "Hello", - timestamp = System.currentTimeMillis(), - processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - ) - val thread = - ThreadEntity( - threadId = threadId, - createdAt = System.currentTimeMillis(), - llmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId = null, - ) - coEvery { observePendingMessagesUseCase(threadId) } returns - flow { - delay(10) - emit(message) - } - coEvery { manageThreadsUseCase.getThread(threadId) } returns flowOf(thread) - coEvery { settingsRepository.geminiApiKey } returns flowOf(null) - coEvery { settingsRepository.disconnectedApps } returns flowOf(emptySet()) + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + + setupDefaultMocks(threadId, message, thread, apiKey = null) + coEvery { getInstalledAppsUseCase() } returns emptyList() agentOrchestrator.observeAndProcessMessages(threadId) @@ -133,34 +122,13 @@ class AgentOrchestratorTest { fun `observeAndProcessMessages fails when LLM returns error`() = runTest { val threadId = "thread_1" - val message = - MessageEntity( - messageId = "msg_1", - threadId = threadId, - role = MessageRole.USER, - textContent = "Hello", - timestamp = System.currentTimeMillis(), - processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - ) - val thread = - ThreadEntity( - threadId = threadId, - createdAt = System.currentTimeMillis(), - llmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId = null, - ) + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) val llmProvider = mockk() - coEvery { observePendingMessagesUseCase(threadId) } returns - flow { - delay(10) - emit(message) - } - coEvery { manageThreadsUseCase.getThread(threadId) } returns flowOf(thread) - coEvery { settingsRepository.geminiApiKey } returns flowOf("dummy_key") - coEvery { settingsRepository.disconnectedApps } returns flowOf(emptySet()) - coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + coEvery { getInstalledAppsUseCase() } returns emptyList() val errorMsg = "LLM failed" coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns @@ -190,38 +158,20 @@ class AgentOrchestratorTest { fun `observeAndProcessMessages succeeds when LLM returns text`() = runTest { val threadId = "thread_1" - val message = - MessageEntity( - messageId = "msg_1", - threadId = threadId, - role = MessageRole.USER, - textContent = "Hello", - timestamp = System.currentTimeMillis(), - processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - ) - val thread = - ThreadEntity( - threadId = threadId, - createdAt = System.currentTimeMillis(), - llmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId = null, - ) + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) val llmProvider = mockk() - coEvery { observePendingMessagesUseCase(threadId) } returns - flow { - delay(10) - emit(message) - } - coEvery { manageThreadsUseCase.getThread(threadId) } returns flowOf(thread) - coEvery { settingsRepository.geminiApiKey } returns flowOf("dummy_key") - coEvery { settingsRepository.disconnectedApps } returns flowOf(emptySet()) - coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + coEvery { getInstalledAppsUseCase() } returns emptyList() val responseText = "Hi there" coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Success("interaction_123", listOf(LlmResponsePart.Text(responseText))) + LlmResponse.Success( + "interaction_123", + listOf(LlmResponsePart.Text(responseText)), + ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -243,4 +193,241 @@ class AgentOrchestratorTest { ) } } + + @Test + fun `observeAndProcessMessages scopes tools by app name`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "@AppFunction Testing Agent run geo code address for n1c4ag") + val thread = createThread(threadId) + val llmProvider = mockk() + + // Mock install apps + val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) + val app2 = AppInfo("com.google.android.digitalwellbeing", "Digital Wellbeing", null) + coEvery { getInstalledAppsUseCase() } returns listOf(app1, app2) + + // Mock tools + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + mockAppFunctions(listOf(tool1, tool2)) + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = any(), + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + + // Run agent orchestrator + agentOrchestrator.observeAndProcessMessages(threadId) + + // Verify correct details + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } + } + + @Test + fun `observeAndProcessMessages scopes tools by app name case insensitively`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "@appfunction testing agent run geo code address for n1c4ag") + val thread = createThread(threadId) + val llmProvider = mockk() + + val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) + coEvery { getInstalledAppsUseCase() } returns listOf(app1) + + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + mockAppFunctions(listOf(tool1)) + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } + } + + @Test + fun `observeAndProcessMessages does not scope tools when app name is unrecognized`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "@UnrecognizedApp run geo code address for n1c4ag") + val thread = createThread(threadId) + val llmProvider = mockk() + + val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) + coEvery { getInstalledAppsUseCase() } returns listOf(app1) + + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + mockAppFunctions(listOf(tool1)) + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("@UnrecognizedApp run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } + } + + @Test + fun `observeAndProcessMessages does not scope tools when no prefix is present`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "run geo code address for n1c4ag") + val thread = createThread(threadId) + val llmProvider = mockk() + + coEvery { getInstalledAppsUseCase() } returns emptyList() + + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + mockAppFunctions(listOf(tool1)) + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify(exactly = 0) { getInstalledAppsUseCase() } + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } + } + + @Test + fun `observeAndProcessMessages scopes tools by app name when input is exactly the app name`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "@AppFunction Testing Agent") + val thread = createThread(threadId) + val llmProvider = mockk() + + val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) + coEvery { getInstalledAppsUseCase() } returns listOf(app1) + + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + mockAppFunctions(listOf(tool1)) + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } + } + + private fun createUserMessage( + threadId: String, + textContent: String, + messageId: String = "message_1", + ) = MessageEntity( + messageId = messageId, + threadId = threadId, + role = MessageRole.USER, + textContent = textContent, + timestamp = System.currentTimeMillis(), + processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, + ) + + private fun createThread( + threadId: String, + llmModel: LlmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, + latestInteractionId: String? = null, + ) = ThreadEntity( + threadId = threadId, + createdAt = System.currentTimeMillis(), + llmModel = llmModel, + latestInteractionId = latestInteractionId, + ) + + private fun createMockTool( + packageName: String, + id: String, + isEnabled: Boolean = true, + ): AppFunctionMetadata { + val tool = mockk() + every { tool.packageName } returns packageName + every { tool.id } returns id + every { tool.isEnabled } returns isEnabled + return tool + } + + private fun mockAppFunctions(tools: List) { + val packageMetadata = mockk(relaxed = true) + coEvery { getAppFunctionsUseCase() } returns flowOf(mapOf(packageMetadata to tools)) + } + + private fun setupDefaultMocks( + threadId: String, + message: MessageEntity, + thread: ThreadEntity, + apiKey: String? = "dummy_key", + disconnectedApps: Set = emptySet(), + llmProvider: LlmProvider = mockk(), + ) { + coEvery { observePendingMessagesUseCase(threadId) } returns + flow { + delay(10) + emit(message) + } + coEvery { manageThreadsUseCase.getThread(threadId) } returns flowOf(thread) + coEvery { settingsRepository.geminiApiKey } returns flowOf(apiKey) + coEvery { settingsRepository.disconnectedApps } returns flowOf(disconnectedApps) + coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider + } } From fa115f6ea1f42807de63fdf9427a82f7a50724b5 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Mon, 15 Jun 2026 13:16:00 +0000 Subject: [PATCH 2/3] agent: Implement autocomplete menu and metadata-backed app scoping Transition the app-scoping feature from the temporary text-parsing bootstrap (b/508130322) to a robust, database-driven metadata architecture (b/521319810). This CL implements metadata-backed tool filtering, a keyboard-friendly autocomplete suggestions menu, and rich visual mention styling. Database & Pipeline: - Add a nullable `targetPackageName: String? = null` column to `MessageEntity` to store the scoped package name securely. - Bump database version to 2 and configure Room with `.fallbackToDestructiveMigration()` in `DataModule.kt` to automatically handle schema updates during development. - Update `SendMessageUseCase.kt` to accept and pass `targetPackageName` to the database. - Simplify `AgentOrchestrator.kt` by completely deleting the obsolete prefix-parsing engine. The orchestrator now reads the package scope directly from database metadata and passes the raw user prompt to the LLM unchanged (leaving the `@AppName` mention in the prompt). UI & Autocomplete Menu: - Refactor the chat input state from `String` to `TextFieldValue` to enable explicit cursor management. - Replace the default `DropdownMenu` with a custom floating `Popup` positioned exactly 2.dp above the `OutlinedTextField` using `PopupPositionProvider` to prevent keyboard overlap. - Support multi-word space searching in the autocomplete query (e.g. typing `@AppFunction Testing Agent I` continues searching and refines the list until no match is found, at which point the dropdown disappears). - Snap the active insertion cursor automatically to the very end of the text field (after the trailing space) upon selecting an autocomplete item to allow seamless typing. - Style active mentions inside the `OutlinedTextField` as bold purple text with no background. - Render historical mentions in the `MessageBubble` chat history as premium rounded chips with exactly 4.dp horizontal padding and 6.dp rounded corners using a custom `drawBehind` Canvas modifier. - Reactively reset the active package scope to null if the user deletes a mention in the input field. ViewModel Refinement: - Inject `GetAppFunctionsUseCase` into `AgentDemoViewModel.kt` and reactively filter the autocomplete app list to ONLY display apps that actually have registered AppFunctions (tools), preventing useless app scopes. Tests & Verification: - Remove obsolete prefix-parsing tests in `AgentOrchestratorTest.kt` and replace them with robust metadata-scoping tests. - Update all ViewModel constructor sites in `AgentDemoViewModelTest.kt` with mocked use cases. Change-Id: I0161976d85433e5a672fc2cd2d95595a1cf930c3 --- .../appfunctions/agent/data/db/AppDatabase.kt | 2 +- .../agent/data/db/entities/MessageEntity.kt | 1 + .../appfunctions/agent/di/DataModule.kt | 1 + .../agent/domain/AgentOrchestrator.kt | 30 +- .../agent/domain/chat/SendMessageUseCase.kt | 2 + .../ui/screens/agentdemo/AgentDemoScreen.kt | 421 +++++++++++++++--- .../screens/agentdemo/AgentDemoViewModel.kt | 25 +- .../ui/screens/agentdemo/AgentUiState.kt | 7 +- .../agent/domain/AgentOrchestratorTest.kt | 142 +----- .../agentdemo/AgentDemoViewModelTest.kt | 27 +- 10 files changed, 442 insertions(+), 216 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt index f6b703a..ae383e6 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt @@ -21,7 +21,7 @@ import com.example.appfunctions.agent.data.db.dao.ChatDao import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity -@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 1, exportSchema = false) +@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 2, exportSchema = false) abstract class AppDatabase : RoomDatabase() { abstract fun chatDao(): ChatDao } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt index 7c252f4..075700b 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt @@ -45,6 +45,7 @@ data class MessageEntity( * only non-null if Assistant returned PendingIntent tool response. */ val pendingIntentId: String? = null, + val targetPackageName: String? = null, ) enum class MessageRole { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 52d4fd0..78ddcc7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -85,6 +85,7 @@ abstract class DataModule { AppDatabase::class.java, "app_database", ) + .fallbackToDestructiveMigration() .build() } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 2d79804..b48931d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -27,7 +27,6 @@ import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFuncti import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase -import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -65,7 +64,6 @@ class AgentOrchestrator private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, - private val getInstalledAppsUseCase: GetInstalledAppsUseCase, ) { private val _status = MutableStateFlow(AgentStatus.Idle) @@ -111,7 +109,8 @@ class AgentOrchestrator val disconnectedApps = settingsRepository.disconnectedApps.first() val allTools = getAppFunctionsUseCase().first().values.flatten() - val (targetPackageName, queryText) = parseTargetPackage(message.textContent) + val targetPackageName = message.targetPackageName + val queryText = message.textContent val tools = filterTools(allTools, disconnectedApps, targetPackageName) @@ -135,31 +134,6 @@ class AgentOrchestrator } } - // TODO: b/521319810 - This text-based parsing is a temporary bootstrapping method implemented in b/508130322. - // In b/521319810, the UI autocomplete menu will return a unique package name - // and attach it directly to the MessageEntity metadata. - // Once that is implemented, this entire `@` parsing block and `getInstalledAppsUseCase` lookup - // should be replaced by reading `targetPackageName` directly from the message metadata. - private suspend fun parseTargetPackage(content: String): Pair { - if (!content.startsWith("@")) { - return Pair(null, content) - } - - val installedApps = getInstalledAppsUseCase() - val sortedApps = installedApps.sortedByDescending { it.label.length } - - for (app in sortedApps) { - val prefix = "@${app.label}" - if (content.equals(prefix, ignoreCase = true)) { - return Pair(app.packageName, "") - } else if (content.startsWith("$prefix ", ignoreCase = true)) { - return Pair(app.packageName, content.substring(prefix.length + 1).trim()) - } - } - - return Pair(null, content) - } - private fun filterTools( allTools: List, disconnectedApps: Set, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt index 1a17d25..91c3518 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt @@ -42,6 +42,7 @@ class SendMessageUseCase textContent: String, processingStatus: MessageProcessingStatus, pendingIntentId: String? = null, + targetPackageName: String? = null, ) { val message = MessageEntity( @@ -52,6 +53,7 @@ class SendMessageUseCase timestamp = System.currentTimeMillis(), processingStatus = processingStatus, pendingIntentId = pendingIntentId, + targetPackageName = targetPackageName, ) chatRepository.sendMessage(message) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 046f594..0a891a6 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -48,6 +48,8 @@ import androidx.compose.material.icons.filled.ArrowDropDown import androidx.compose.material.icons.filled.Menu import androidx.compose.material.icons.filled.Warning import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.DrawerState import androidx.compose.material3.DrawerValue @@ -76,6 +78,10 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.drawBehind +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.input.key.Key @@ -85,10 +91,30 @@ import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.platform.LocalFocusManager import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.text.SpanStyle +import androidx.compose.ui.text.TextLayoutResult +import androidx.compose.ui.text.TextRange +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.input.OffsetMapping +import androidx.compose.ui.text.input.TextFieldValue +import androidx.compose.ui.text.input.TransformedText +import androidx.compose.ui.text.input.VisualTransformation import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.unit.IntOffset +import androidx.compose.ui.unit.IntRect +import androidx.compose.ui.unit.IntSize +import androidx.compose.ui.unit.LayoutDirection import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.window.Popup +import androidx.compose.ui.window.PopupPositionProvider +import androidx.compose.ui.window.PopupProperties import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -99,6 +125,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.ui.screens.debugging.LazyExposedDropdownMenu import com.mikepenz.markdown.m3.Markdown import kotlinx.coroutines.CoroutineScope @@ -136,6 +163,7 @@ fun AgentDemoContent( is AgentUiState.Loading -> { AgentDemoLoadingScreen() } + is AgentUiState.Loaded -> { AgentDemoLoadedScreen( uiState = uiState, @@ -196,8 +224,16 @@ fun AgentDemoLoadedScreen( packageManager: PackageManager, initialSidePanelVisible: Boolean = false, ) { - var messageText by remember { mutableStateOf("") } + var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } + var selectedAppPackageName by remember { mutableStateOf(null) } + + val chipBgColor = MaterialTheme.colorScheme.primaryContainer + val chipTextColor = MaterialTheme.colorScheme.onPrimaryContainer + val visualTransformation = + remember(uiState.installedApps, chipTextColor) { + InlineAppScopingVisualTransformation(uiState.installedApps, chipTextColor) + } Scaffold( modifier = Modifier.fillMaxSize(), @@ -208,7 +244,10 @@ fun AgentDemoLoadedScreen( verticalAlignment = Alignment.CenterVertically, ) { ModelDropdown( - modifier = Modifier.weight(1f).padding(horizontal = 8.dp), + modifier = + Modifier + .weight(1f) + .padding(horizontal = 8.dp), currentThread = uiState.currentThread, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { @@ -232,7 +271,8 @@ fun AgentDemoLoadedScreen( ) { paddingValues -> Row( modifier = - Modifier.fillMaxSize() + Modifier + .fillMaxSize() .imePadding() .padding( top = paddingValues.calculateTopPadding(), @@ -256,13 +296,19 @@ fun AgentDemoLoadedScreen( // Main Chat Area Column( modifier = - Modifier.weight(1f).fillMaxHeight().padding(start = 16.dp, end = 16.dp), + Modifier + .weight(1f) + .fillMaxHeight() + .padding(start = 16.dp, end = 16.dp), verticalArrangement = Arrangement.SpaceBetween, ) { // Messages List LazyColumn( modifier = - Modifier.weight(1f).fillMaxWidth().clip(RoundedCornerShape(16.dp)), + Modifier + .weight(1f) + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)), reverseLayout = true, ) { // Status item at the bottom (above input) if not @@ -284,60 +330,160 @@ fun AgentDemoLoadedScreen( message = message, isValidAction = message.pendingIntentId in uiState.activePendingActionIds, + installedApps = uiState.installedApps, onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) }, ) } } val sendMessage = { - if (messageText.isNotBlank() && uiState.status == AgentStatus.Idle) { - onEvent(AgentUiEvent.OnSendMessage(messageText)) - messageText = "" + val textStr = messageText.text + if (textStr.isNotBlank() && uiState.status == AgentStatus.Idle) { + onEvent(AgentUiEvent.OnSendMessage(textStr, selectedAppPackageName)) + messageText = TextFieldValue("") + selectedAppPackageName = null } } + val textStr = messageText.text + val showAutocomplete = textStr.contains("@") && selectedAppPackageName == null + val autocompleteQuery = + if (showAutocomplete) { + textStr.substringAfterLast("@") + } else { + "" + } + val filteredApps = + remember(autocompleteQuery, uiState.installedApps) { + if (autocompleteQuery.isEmpty()) { + uiState.installedApps + } else { + uiState.installedApps.filter { + it.label.contains(autocompleteQuery, ignoreCase = true) + } + } + } + + val density = LocalDensity.current + val popupPositionProvider = + remember(density) { + object : PopupPositionProvider { + override fun calculatePosition( + anchorBounds: IntRect, + windowSize: IntSize, + layoutDirection: LayoutDirection, + popupContentSize: IntSize, + ): IntOffset { + val gap = with(density) { 2.dp.roundToPx() } + return IntOffset( + x = anchorBounds.left, + y = anchorBounds.top - popupContentSize.height - gap, + ) + } + } + } + + val appMentionRegex = + remember(uiState.installedApps) { + if (uiState.installedApps.isNotEmpty()) { + val appLabelsPattern = + uiState.installedApps.joinToString("|") { Regex.escape(it.label) } + Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + } else { + null + } + } + // Input area - OutlinedTextField( - value = messageText, - onValueChange = { messageText = it }, - modifier = - Modifier.fillMaxWidth().padding(vertical = 16.dp).onPreviewKeyEvent { - keyEvent -> - if ( - (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && - keyEvent.type == KeyEventType.KeyDown + Box(modifier = Modifier.fillMaxWidth()) { + OutlinedTextField( + value = messageText, + onValueChange = { newValue -> + messageText = newValue + val currentText = newValue.text + if (selectedAppPackageName != null && appMentionRegex != null) { + if (!appMentionRegex.containsMatchIn(currentText)) { + selectedAppPackageName = null + } + } + }, + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 16.dp) + .onPreviewKeyEvent { keyEvent -> + if ( + (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && + keyEvent.type == KeyEventType.KeyDown + ) { + sendMessage() + true + } else { + false + } + }, + enabled = uiState.status == AgentStatus.Idle, + shape = CircleShape, + placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, + visualTransformation = visualTransformation, + colors = + OutlinedTextFieldDefaults.colors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + unfocusedBorderColor = Color.Transparent, + focusedBorderColor = Color.Transparent, + ), + trailingIcon = { + IconButton( + onClick = sendMessage, + enabled = + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle, ) { - sendMessage() - true - } else { - false + Icon( + imageVector = Icons.AutoMirrored.Filled.Send, + contentDescription = + stringResource(R.string.agent_demo_send), + ) } }, - enabled = uiState.status == AgentStatus.Idle, - shape = CircleShape, - placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, - colors = - OutlinedTextFieldDefaults.colors( - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - unfocusedBorderColor = Color.Transparent, - focusedBorderColor = Color.Transparent, - ), - trailingIcon = { - IconButton( - onClick = sendMessage, - enabled = - messageText.isNotBlank() && - uiState.status == AgentStatus.Idle, + ) + + if (showAutocomplete && filteredApps.isNotEmpty()) { + Popup( + popupPositionProvider = popupPositionProvider, + onDismissRequest = {}, + properties = PopupProperties(focusable = false), ) { - Icon( - imageVector = Icons.AutoMirrored.Filled.Send, - contentDescription = - stringResource(R.string.agent_demo_send), - ) + Card( + modifier = Modifier.fillMaxWidth(0.9f), + elevation = CardDefaults.cardElevation(defaultElevation = 8.dp), + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceBright), + shape = MaterialTheme.shapes.medium, + ) { + Column(modifier = Modifier.padding(vertical = 4.dp)) { + filteredApps.take(5).forEach { app -> + DropdownMenuItem( + text = { Text(app.label) }, + onClick = { + val currentText = messageText.text + val textBeforeMention = + currentText.substringBeforeLast("@") + val newText = "$textBeforeMention@${app.label} " + messageText = + TextFieldValue( + text = newText, + selection = TextRange(newText.length), + ) + selectedAppPackageName = app.packageName + }, + ) + } + } + } } - }, - ) + } + } } } } @@ -374,7 +520,11 @@ fun ModelDropdown( } Row( - modifier = Modifier.fillMaxWidth().height(56.dp).padding(start = 4.dp, end = 16.dp), + modifier = + Modifier + .fillMaxWidth() + .height(56.dp) + .padding(start = 4.dp, end = 16.dp), verticalAlignment = Alignment.CenterVertically, ) { IconButton(onClick = onMenuClick) { @@ -382,7 +532,8 @@ fun ModelDropdown( } Row( modifier = - Modifier.weight(1f) + Modifier + .weight(1f) .fillMaxHeight() .menuAnchor( ExposedDropdownMenuAnchorType.PrimaryEditable, @@ -448,6 +599,7 @@ fun ModelDropdown( fun MessageBubble( message: MessageEntity, isValidAction: Boolean, + installedApps: List, onConfirmAction: (String) -> Unit, ) { val alignment = if (message.role == MessageRole.USER) Alignment.End else Alignment.Start @@ -466,7 +618,10 @@ fun MessageBubble( } Column( - modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp, horizontal = 2.dp), + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp, horizontal = 2.dp), horizontalAlignment = alignment, ) { Surface( @@ -495,10 +650,52 @@ fun MessageBubble( if (message.role != MessageRole.USER) { Markdown(content = contentText) } else { + val chipBgColor = MaterialTheme.colorScheme.primary + val chipTextColor = MaterialTheme.colorScheme.onPrimary + val formattedText = + remember(contentText, installedApps, chipTextColor) { + formatMessageText(contentText, installedApps, chipTextColor) + } + var textLayoutResult by remember { mutableStateOf(null) } + Text( - text = contentText, + text = formattedText, color = textColor, style = MaterialTheme.typography.bodyLarge, + onTextLayout = { textLayoutResult = it }, + modifier = + Modifier.drawBehind { + val layout = textLayoutResult ?: return@drawBehind + formattedText.getStringAnnotations( + tag = "mention", + start = 0, + end = formattedText.length, + ) + .forEach { annotation -> + val start = annotation.start + val end = annotation.end + val path = layout.getPathForRange(start, end) + val rect = path.getBounds() + + val paddingPx = 4.dp.toPx() + val cornerRadiusPx = 6.dp.toPx() + + drawRoundRect( + color = chipBgColor, + topLeft = Offset(rect.left - paddingPx, rect.top), + size = + Size( + rect.width + (paddingPx * 2), + rect.height, + ), + cornerRadius = + CornerRadius( + cornerRadiusPx, + cornerRadiusPx, + ), + ) + } + }, ) } } @@ -537,7 +734,10 @@ fun StatusIndicator( when (status) { AgentStatus.Thinking -> { Row( - modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp), + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), verticalAlignment = Alignment.CenterVertically, ) { CircularProgressIndicator(modifier = Modifier.size(24.dp)) @@ -548,6 +748,7 @@ fun StatusIndicator( ) } } + is AgentStatus.InvokingTool -> { val appName = try { @@ -564,7 +765,10 @@ fun StatusIndicator( } Surface( - modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp), + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), shape = MaterialTheme.shapes.large, color = MaterialTheme.colorScheme.surfaceBright, shadowElevation = 2.dp, @@ -595,6 +799,7 @@ fun StatusIndicator( } } } + AgentStatus.Idle -> { // Nothing to show } @@ -608,7 +813,13 @@ fun ChatHistorySidePanel( onEvent: (AgentUiEvent) -> Unit, modifier: Modifier = Modifier, ) { - Column(modifier = modifier.width(280.dp).fillMaxHeight().padding(16.dp)) { + Column( + modifier = + modifier + .width(280.dp) + .fillMaxHeight() + .padding(16.dp), + ) { Text( text = stringResource(R.string.agent_demo_chat_history), style = MaterialTheme.typography.titleLarge, @@ -637,9 +848,12 @@ fun ChatHistorySidePanel( Surface( modifier = - Modifier.fillMaxWidth().padding(vertical = 4.dp).clickable { - onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) - }, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp) + .clickable { + onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) + }, shape = MaterialTheme.shapes.medium, color = backgroundColor, contentColor = textColor, @@ -661,3 +875,102 @@ fun ChatHistorySidePanel( } } } + +class InlineAppScopingVisualTransformation( + private val installedApps: List, + private val chipTextColor: Color, +) : VisualTransformation { + private val regex: Regex? = + if (installedApps.isNotEmpty()) { + val appLabelsPattern = installedApps.joinToString("|") { Regex.escape(it.label) } + Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + } else { + null + } + + override fun filter(text: AnnotatedString): TransformedText { + val rawText = text.text + val currentRegex = regex + if (currentRegex == null || !rawText.contains("@")) { + return TransformedText(text, OffsetMapping.Identity) + } + + val matches = currentRegex.findAll(rawText) + + val annotatedString = + buildAnnotatedString { + var lastIndex = 0 + matches.forEach { match -> + append(rawText.substring(lastIndex, match.range.first)) + pushStringAnnotation(tag = "mention", annotation = match.value) + withStyle( + SpanStyle( + color = chipTextColor, + fontWeight = FontWeight.Bold, + ), + ) { + append(match.value) + } + pop() + lastIndex = match.range.last + 1 + } + if (lastIndex < rawText.length) { + append(rawText.substring(lastIndex)) + } + } + return TransformedText(annotatedString, OffsetMapping.Identity) + } +} + +fun formatMessageText( + text: String, + installedApps: List, + chipTextColor: Color, +): AnnotatedString { + if (installedApps.isEmpty() || !text.contains("@")) { + return AnnotatedString(text) + } + val appLabelsPattern = installedApps.joinToString("|") { Regex.escape(it.label) } + val regex = Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + val matches = regex.findAll(text) + + return buildAnnotatedString { + var lastIndex = 0 + matches.forEach { match -> + val precedingText = text.substring(lastIndex, match.range.first) + if (precedingText.isNotEmpty()) { + append(precedingText) + } + + pushStringAnnotation(tag = "mention", annotation = match.value) + val appName = match.value + if (appName.isNotEmpty()) { + val mainPart = appName.dropLast(1) + val lastChar = appName.takeLast(1) + + withStyle( + SpanStyle( + color = chipTextColor, + fontWeight = FontWeight.Bold, + ), + ) { + append(mainPart) + } + withStyle( + SpanStyle( + color = chipTextColor, + fontWeight = FontWeight.Bold, + letterSpacing = 3.sp, + ), + ) { + append(lastChar) + } + } + pop() + lastIndex = match.range.last + 1 + } + if (lastIndex < text.length) { + append(text.substring(lastIndex)) + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index f20e972..06ba145 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -44,6 +44,10 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import javax.inject.Inject +import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase +import com.example.appfunctions.agent.domain.appfunction.AppInfo +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase + @HiltViewModel class AgentDemoViewModel @Inject @@ -57,13 +61,25 @@ class AgentDemoViewModel private val observeActivePendingIntentsUseCase: ObserveActivePendingIntentsUseCase, private val launchPendingIntentUseCase: LaunchPendingIntentUseCase, private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, + private val getInstalledAppsUseCase: GetInstalledAppsUseCase, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase, ) : ViewModel() { private val _uiState = MutableStateFlow(AgentUiState.Loading) val uiState: StateFlow = _uiState.asStateFlow() + private val _installedApps = MutableStateFlow>(emptyList()) + private var observationJob: Job? = null init { + viewModelScope.launch { + getAppFunctionsUseCase().collect { toolsMap -> + val allTools = toolsMap.values.flatten() + val packagesWithTools = allTools.map { it.packageName }.toSet() + _installedApps.value = getInstalledAppsUseCase().filter { it.packageName in packagesWithTools } + } + } + viewModelScope.launch { observeActivePendingIntentsUseCase().collect { activePendingActionIds -> val currentState = _uiState.value @@ -87,15 +103,17 @@ class AgentDemoViewModel settingsRepository.selectedProvider, agentOrchestrator.status, savedStateHandle.getStateFlow(MainActivity.ARG_THREAD_ID, null), + _installedApps, ) { threads, provider, status, targetThreadId, + apps, -> - ThreadConfig(threads, provider, status, targetThreadId) + ThreadConfig(threads, provider, status, targetThreadId, apps) } - .collectLatest { (threads, provider, status, targetThreadId) -> + .collectLatest { (threads, provider, status, targetThreadId, apps) -> val currentThread = threads.find { it.threadId == targetThreadId } ?: threads.firstOrNull() @@ -116,6 +134,7 @@ class AgentDemoViewModel threads = threads, activePendingActionIds = currentLoadedState?.activePendingActionIds ?: emptySet(), + installedApps = apps, ) // Start observing messages for the current thread if not already doing so @@ -156,6 +175,7 @@ class AgentDemoViewModel role = MessageRole.USER, textContent = event.text, processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, + targetPackageName = event.targetPackageName, ) } } @@ -196,4 +216,5 @@ private data class ThreadConfig( val provider: LlmProviderName, val status: AgentStatus, val targetThreadId: String?, + val installedApps: List, ) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt index 0be61ea..4798f79 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt @@ -19,6 +19,7 @@ import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.appfunction.AppInfo /** Represents the UI state for the Agent Demo screen. */ sealed class AgentUiState { @@ -30,12 +31,16 @@ sealed class AgentUiState { val status: AgentStatus = AgentStatus.Idle, val threads: List = emptyList(), val activePendingActionIds: Set = emptySet(), + val installedApps: List = emptyList(), ) : AgentUiState() } /** Represents UI events for the Agent Demo screen. */ sealed class AgentUiEvent { - data class OnSendMessage(val text: String) : AgentUiEvent() + data class OnSendMessage( + val text: String, + val targetPackageName: String? = null, + ) : AgentUiEvent() data class OnModelSelected(val model: LlmModel) : AgentUiEvent() diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 0543f27..592d28d 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -24,11 +24,9 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity -import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase -import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -65,8 +63,6 @@ class AgentOrchestratorTest { private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) - private val getInstalledAppsUseCase: GetInstalledAppsUseCase = mockk() - private lateinit var agentOrchestrator: AgentOrchestrator @Before @@ -84,7 +80,6 @@ class AgentOrchestratorTest { convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, savePendingIntentUseCase = savePendingIntentUseCase, - getInstalledAppsUseCase = getInstalledAppsUseCase, ) } @@ -96,7 +91,6 @@ class AgentOrchestratorTest { val thread = createThread(threadId) setupDefaultMocks(threadId, message, thread, apiKey = null) - coEvery { getInstalledAppsUseCase() } returns emptyList() agentOrchestrator.observeAndProcessMessages(threadId) @@ -128,7 +122,6 @@ class AgentOrchestratorTest { setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - coEvery { getInstalledAppsUseCase() } returns emptyList() val errorMsg = "LLM failed" coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns @@ -164,7 +157,6 @@ class AgentOrchestratorTest { setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - coEvery { getInstalledAppsUseCase() } returns emptyList() val responseText = "Hi there" coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns @@ -195,66 +187,24 @@ class AgentOrchestratorTest { } @Test - fun `observeAndProcessMessages scopes tools by app name`() = + fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = runTest { val threadId = "thread_1" - val message = createUserMessage(threadId, "@AppFunction Testing Agent run geo code address for n1c4ag") + val message = + createUserMessage( + threadId = threadId, + textContent = "run geo code address for n1c4ag", + targetPackageName = "com.google.android.appfunctiontestingagent", + ) val thread = createThread(threadId) val llmProvider = mockk() - // Mock install apps - val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) - val app2 = AppInfo("com.google.android.digitalwellbeing", "Digital Wellbeing", null) - coEvery { getInstalledAppsUseCase() } returns listOf(app1, app2) - - // Mock tools val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") mockAppFunctions(listOf(tool1, tool2)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { - llmProvider.generateResponse( - previousInteractionId = any(), - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - - // Run agent orchestrator - agentOrchestrator.observeAndProcessMessages(threadId) - - // Verify correct details - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any(), - ) - } - } - - @Test - fun `observeAndProcessMessages scopes tools by app name case insensitively`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "@appfunction testing agent run geo code address for n1c4ag") - val thread = createThread(threadId) - val llmProvider = mockk() - - val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) - coEvery { getInstalledAppsUseCase() } returns listOf(app1) - - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - mockAppFunctions(listOf(tool1)) - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) @@ -273,50 +223,16 @@ class AgentOrchestratorTest { } @Test - fun `observeAndProcessMessages does not scope tools when app name is unrecognized`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "@UnrecognizedApp run geo code address for n1c4ag") - val thread = createThread(threadId) - val llmProvider = mockk() - - val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) - coEvery { getInstalledAppsUseCase() } returns listOf(app1) - - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - mockAppFunctions(listOf(tool1)) - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - - agentOrchestrator.observeAndProcessMessages(threadId) - - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("@UnrecognizedApp run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any(), - ) - } - } - - @Test - fun `observeAndProcessMessages does not scope tools when no prefix is present`() = + fun `observeAndProcessMessages does not scope tools when targetPackageName is null`() = runTest { val threadId = "thread_1" val message = createUserMessage(threadId, "run geo code address for n1c4ag") val thread = createThread(threadId) val llmProvider = mockk() - coEvery { getInstalledAppsUseCase() } returns emptyList() - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - mockAppFunctions(listOf(tool1)) + val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + mockAppFunctions(listOf(tool1, tool2)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) @@ -326,45 +242,11 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) - coVerify(exactly = 0) { getInstalledAppsUseCase() } coVerify { llmProvider.generateResponse( previousInteractionId = null, input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any(), - ) - } - } - - @Test - fun `observeAndProcessMessages scopes tools by app name when input is exactly the app name`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "@AppFunction Testing Agent") - val thread = createThread(threadId) - val llmProvider = mockk() - - val app1 = AppInfo("com.google.android.appfunctiontestingagent", "AppFunction Testing Agent", null) - coEvery { getInstalledAppsUseCase() } returns listOf(app1) - - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - mockAppFunctions(listOf(tool1)) - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - - agentOrchestrator.observeAndProcessMessages(threadId) - - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("")), - tools = listOf(tool1), + tools = listOf(tool1, tool2), apiKey = "dummy_key", modelName = any(), ) @@ -375,6 +257,7 @@ class AgentOrchestratorTest { threadId: String, textContent: String, messageId: String = "message_1", + targetPackageName: String? = null, ) = MessageEntity( messageId = messageId, threadId = threadId, @@ -382,6 +265,7 @@ class AgentOrchestratorTest { textContent = textContent, timestamp = System.currentTimeMillis(), processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, + targetPackageName = targetPackageName, ) private fun createThread( diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt index 6834f56..27a4805 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt @@ -25,6 +25,8 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -38,6 +40,7 @@ import io.mockk.mockk import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.resetMain import kotlinx.coroutines.test.runTest @@ -60,6 +63,8 @@ class AgentDemoViewModelTest { private lateinit var observeActivePendingIntentsUseCase: ObserveActivePendingIntentsUseCase private lateinit var launchPendingIntentUseCase: LaunchPendingIntentUseCase private lateinit var consumePendingIntentUseCase: ConsumePendingIntentUseCase + private lateinit var getInstalledAppsUseCase: GetInstalledAppsUseCase + private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase private lateinit var viewModel: AgentDemoViewModel @@ -82,6 +87,11 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase = mockk() launchPendingIntentUseCase = mockk() consumePendingIntentUseCase = mockk(relaxed = true) + getInstalledAppsUseCase = mockk() + getAppFunctionsUseCase = mockk() + + every { getInstalledAppsUseCase() } returns emptyList() + every { getAppFunctionsUseCase() } returns flowOf(emptyMap()) every { manageThreadsUseCase.getThreads() } returns threadsFlow every { settingsRepository.selectedProvider } returns selectedProviderFlow @@ -124,6 +134,8 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW)) @@ -150,6 +162,8 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } @@ -179,10 +193,15 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } - assertEquals(existingThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) + assertEquals( + existingThread, + (viewModel.uiState.value as AgentUiState.Loaded).currentThread, + ) } @Test @@ -209,6 +228,8 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit @@ -249,6 +270,8 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) val message = @@ -296,6 +319,8 @@ class AgentDemoViewModelTest { observeActivePendingIntentsUseCase, launchPendingIntentUseCase, consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, ) assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) From e1879c8bda486b44d602ef88b28e314e7d1c1fb4 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 17 Jun 2026 22:28:28 +0000 Subject: [PATCH 3/3] fix: address gemini-code-assist bot code review feedback Change-Id: Iabacf439b9bb753325399289792f2a5138c4cd72 --- .../ui/screens/agentdemo/AgentDemoScreen.kt | 469 +++++++++--------- .../screens/agentdemo/AgentDemoViewModel.kt | 276 ++++++----- 2 files changed, 388 insertions(+), 357 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 0a891a6..853e73d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -41,6 +41,8 @@ import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.InlineTextContent +import androidx.compose.foundation.text.appendInlineContent import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add @@ -78,10 +80,6 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip -import androidx.compose.ui.draw.drawBehind -import androidx.compose.ui.geometry.CornerRadius -import androidx.compose.ui.geometry.Offset -import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.input.key.Key @@ -95,8 +93,9 @@ import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.platform.LocalFocusManager import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.text.Placeholder +import androidx.compose.ui.text.PlaceholderVerticalAlign import androidx.compose.ui.text.SpanStyle -import androidx.compose.ui.text.TextLayoutResult import androidx.compose.ui.text.TextRange import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.font.FontWeight @@ -104,6 +103,7 @@ import androidx.compose.ui.text.input.OffsetMapping import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.text.input.TransformedText import androidx.compose.ui.text.input.VisualTransformation +import androidx.compose.ui.text.rememberTextMeasurer import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.text.withStyle import androidx.compose.ui.unit.IntOffset @@ -111,7 +111,6 @@ import androidx.compose.ui.unit.IntRect import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.LayoutDirection import androidx.compose.ui.unit.dp -import androidx.compose.ui.unit.sp import androidx.compose.ui.window.Popup import androidx.compose.ui.window.PopupPositionProvider import androidx.compose.ui.window.PopupProperties @@ -143,7 +142,7 @@ fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { fun AgentDemoContent( uiState: AgentUiState, onEvent: (AgentUiEvent) -> Unit, - initialSidePanelVisible: Boolean = false, + initialSidePanelVisible: Boolean = false ) { val context = LocalContext.current val packageManager = context.packageManager @@ -172,7 +171,7 @@ fun AgentDemoContent( drawerState = drawerState, scope = scope, packageManager = packageManager, - initialSidePanelVisible = initialSidePanelVisible, + initialSidePanelVisible = initialSidePanelVisible ) } } @@ -188,7 +187,7 @@ fun AgentDemoContent( drawerState = drawerState, drawerContent = { ModalDrawerSheet( - drawerContainerColor = MaterialTheme.colorScheme.surface, + drawerContainerColor = MaterialTheme.colorScheme.surface ) { ChatHistorySidePanel( threads = threads, @@ -196,10 +195,10 @@ fun AgentDemoContent( onEvent = { event -> onEvent(event) scope.launch { drawerState.close() } - }, + } ) } - }, + } ) { content() } @@ -222,7 +221,7 @@ fun AgentDemoLoadedScreen( drawerState: DrawerState, scope: CoroutineScope, packageManager: PackageManager, - initialSidePanelVisible: Boolean = false, + initialSidePanelVisible: Boolean = false ) { var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } @@ -241,13 +240,13 @@ fun AgentDemoLoadedScreen( topBar = { Row( modifier = Modifier.padding(horizontal = 8.dp, vertical = 16.dp), - verticalAlignment = Alignment.CenterVertically, + verticalAlignment = Alignment.CenterVertically ) { ModelDropdown( modifier = - Modifier - .weight(1f) - .padding(horizontal = 8.dp), + Modifier + .weight(1f) + .padding(horizontal = 8.dp), currentThread = uiState.currentThread, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { @@ -256,39 +255,39 @@ fun AgentDemoLoadedScreen( } else { scope.launch { drawerState.open() } } - }, + } ) IconButton( onClick = { onEvent(AgentUiEvent.OnCreateThread(uiState.currentThread.llmModel)) }, - modifier = Modifier.padding(horizontal = 8.dp), + modifier = Modifier.padding(horizontal = 8.dp) ) { Icon(imageVector = Icons.Default.Add, contentDescription = "Create Thread") } } - }, + } ) { paddingValues -> Row( modifier = - Modifier - .fillMaxSize() - .imePadding() - .padding( - top = paddingValues.calculateTopPadding(), - ), + Modifier + .fillMaxSize() + .imePadding() + .padding( + top = paddingValues.calculateTopPadding() + ) ) { // Side Panel (only for wide screens) if (isWideScreen) { AnimatedVisibility( visible = isSidePanelVisible, enter = slideInHorizontally() + expandHorizontally(), - exit = slideOutHorizontally() + shrinkHorizontally(), + exit = slideOutHorizontally() + shrinkHorizontally() ) { ChatHistorySidePanel( threads = uiState.threads, currentThread = uiState.currentThread, - onEvent = onEvent, + onEvent = onEvent ) } } @@ -296,20 +295,20 @@ fun AgentDemoLoadedScreen( // Main Chat Area Column( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .padding(start = 16.dp, end = 16.dp), - verticalArrangement = Arrangement.SpaceBetween, + Modifier + .weight(1f) + .fillMaxHeight() + .padding(start = 16.dp, end = 16.dp), + verticalArrangement = Arrangement.SpaceBetween ) { // Messages List LazyColumn( modifier = - Modifier - .weight(1f) - .fillMaxWidth() - .clip(RoundedCornerShape(16.dp)), - reverseLayout = true, + Modifier + .weight(1f) + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)), + reverseLayout = true ) { // Status item at the bottom (above input) if not // idle @@ -317,21 +316,21 @@ fun AgentDemoLoadedScreen( item { StatusIndicator( status = uiState.status, - packageManager = packageManager, + packageManager = packageManager ) } } items( items = uiState.messages.reversed(), - key = { message -> message.messageId }, + key = { message -> message.messageId } ) { message -> MessageBubble( message = message, isValidAction = - message.pendingIntentId in uiState.activePendingActionIds, + message.pendingIntentId in uiState.activePendingActionIds, installedApps = uiState.installedApps, - onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) }, + onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) } ) } } @@ -346,10 +345,13 @@ fun AgentDemoLoadedScreen( } val textStr = messageText.text - val showAutocomplete = textStr.contains("@") && selectedAppPackageName == null + val lastAtIndex = textStr.lastIndexOf('@') + val showAutocomplete = lastAtIndex >= 0 && + (lastAtIndex == 0 || textStr[lastAtIndex - 1].isWhitespace()) && + selectedAppPackageName == null val autocompleteQuery = if (showAutocomplete) { - textStr.substringAfterLast("@") + textStr.substring(lastAtIndex + 1) } else { "" } @@ -372,12 +374,12 @@ fun AgentDemoLoadedScreen( anchorBounds: IntRect, windowSize: IntSize, layoutDirection: LayoutDirection, - popupContentSize: IntSize, + popupContentSize: IntSize ): IntOffset { val gap = with(density) { 2.dp.roundToPx() } return IntOffset( x = anchorBounds.left, - y = anchorBounds.top - popupContentSize.height - gap, + y = anchorBounds.top - popupContentSize.height - gap ) } } @@ -408,58 +410,60 @@ fun AgentDemoLoadedScreen( } }, modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 16.dp) - .onPreviewKeyEvent { keyEvent -> - if ( - (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && - keyEvent.type == KeyEventType.KeyDown - ) { - sendMessage() - true - } else { - false - } - }, + Modifier + .fillMaxWidth() + .padding(vertical = 16.dp) + .onPreviewKeyEvent { keyEvent -> + if ( + (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && + keyEvent.type == KeyEventType.KeyDown + ) { + sendMessage() + true + } else { + false + } + }, enabled = uiState.status == AgentStatus.Idle, shape = CircleShape, placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, visualTransformation = visualTransformation, colors = - OutlinedTextFieldDefaults.colors( - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - unfocusedBorderColor = Color.Transparent, - focusedBorderColor = Color.Transparent, - ), + OutlinedTextFieldDefaults.colors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + unfocusedBorderColor = Color.Transparent, + focusedBorderColor = Color.Transparent + ), trailingIcon = { IconButton( onClick = sendMessage, enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle, + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle ) { Icon( imageVector = Icons.AutoMirrored.Filled.Send, contentDescription = - stringResource(R.string.agent_demo_send), + stringResource(R.string.agent_demo_send) ) } - }, + } ) if (showAutocomplete && filteredApps.isNotEmpty()) { Popup( popupPositionProvider = popupPositionProvider, onDismissRequest = {}, - properties = PopupProperties(focusable = false), + properties = PopupProperties(focusable = false) ) { Card( modifier = Modifier.fillMaxWidth(0.9f), elevation = CardDefaults.cardElevation(defaultElevation = 8.dp), - colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceBright), - shape = MaterialTheme.shapes.medium, + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceBright + ), + shape = MaterialTheme.shapes.medium ) { Column(modifier = Modifier.padding(vertical = 4.dp)) { filteredApps.take(5).forEach { app -> @@ -467,16 +471,34 @@ fun AgentDemoLoadedScreen( text = { Text(app.label) }, onClick = { val currentText = messageText.text - val textBeforeMention = - currentText.substringBeforeLast("@") - val newText = "$textBeforeMention@${app.label} " - messageText = - TextFieldValue( - text = newText, - selection = TextRange(newText.length), - ) - selectedAppPackageName = app.packageName - }, + val selectionStart = messageText.selection.start + val textBeforeCursor = currentText.take( + selectionStart + ) + val textAfterCursor = currentText.drop( + selectionStart + ) + val mentionIndex = textBeforeCursor.lastIndexOf('@') + if (mentionIndex >= 0) { + val textBeforeMention = + textBeforeCursor.substring( + 0, + mentionIndex + ) + val newText = + "$textBeforeMention@${app.label} $textAfterCursor" + val newCursorPosition = + mentionIndex + app.label.length + 2 + messageText = + TextFieldValue( + text = newText, + selection = TextRange( + newCursorPosition + ) + ) + selectedAppPackageName = app.packageName + } + } ) } } @@ -495,19 +517,19 @@ fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, onModelSelected: (LlmModel) -> Unit, - onMenuClick: () -> Unit, + onMenuClick: () -> Unit ) { var expanded by remember { mutableStateOf(false) } ExposedDropdownMenuBox( modifier = modifier, expanded = expanded, - onExpandedChange = { expanded = !expanded }, + onExpandedChange = { expanded = !expanded } ) { Surface( modifier = Modifier.padding(bottom = 8.dp), shadowElevation = 2.dp, shape = CircleShape, - color = MaterialTheme.colorScheme.surfaceBright, + color = MaterialTheme.colorScheme.surfaceBright ) { val text = currentThread?.llmModel?.modelName @@ -521,38 +543,38 @@ fun ModelDropdown( Row( modifier = - Modifier - .fillMaxWidth() - .height(56.dp) - .padding(start = 4.dp, end = 16.dp), - verticalAlignment = Alignment.CenterVertically, + Modifier + .fillMaxWidth() + .height(56.dp) + .padding(start = 4.dp, end = 16.dp), + verticalAlignment = Alignment.CenterVertically ) { IconButton(onClick = onMenuClick) { Icon(imageVector = Icons.Default.Menu, contentDescription = "Menu") } Row( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .menuAnchor( - ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true, - ), - verticalAlignment = Alignment.CenterVertically, + Modifier + .weight(1f) + .fillMaxHeight() + .menuAnchor( + ExposedDropdownMenuAnchorType.PrimaryEditable, + enabled = true + ), + verticalAlignment = Alignment.CenterVertically ) { Column(modifier = Modifier.weight(1f)) { Text( text = stringResource(R.string.agent_demo_title), style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, + color = MaterialTheme.colorScheme.onSurfaceVariant ) Text( text = text, style = MaterialTheme.typography.bodyMedium, color = textColor, maxLines = 1, - overflow = TextOverflow.Ellipsis, + overflow = TextOverflow.Ellipsis ) } Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) @@ -565,21 +587,21 @@ fun ModelDropdown( onDismissRequest = { expanded = false }, modifier = Modifier.exposedDropdownSize(), containerColor = MaterialTheme.colorScheme.surfaceBright, - shape = RoundedCornerShape(28.dp), + shape = RoundedCornerShape(28.dp) ) { item { Text( "--- Gemini ---", color = MaterialTheme.colorScheme.secondary, modifier = Modifier.padding(horizontal = 24.dp, vertical = 8.dp), - style = MaterialTheme.typography.labelLarge, + style = MaterialTheme.typography.labelLarge ) } val models = listOf( LlmModel.GEMINI_3_1_PRO_PREVIEW, LlmModel.GEMINI_3_FLASH_PREVIEW, - LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW, + LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW ) items(models) { model -> DropdownMenuItem( @@ -588,7 +610,7 @@ fun ModelDropdown( onModelSelected(model) expanded = false }, - contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp), + contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp) ) } } @@ -600,7 +622,7 @@ fun MessageBubble( message: MessageEntity, isValidAction: Boolean, installedApps: List, - onConfirmAction: (String) -> Unit, + onConfirmAction: (String) -> Unit ) { val alignment = if (message.role == MessageRole.USER) Alignment.End else Alignment.Start val isError = message.processingStatus == MessageProcessingStatus.FAILED @@ -619,15 +641,15 @@ fun MessageBubble( Column( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp, horizontal = 2.dp), - horizontalAlignment = alignment, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp, horizontal = 2.dp), + horizontalAlignment = alignment ) { Surface( shape = MaterialTheme.shapes.large, color = backgroundColor, - shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp, + shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp ) { Column(modifier = Modifier.padding(12.dp)) { Row(verticalAlignment = Alignment.CenterVertically) { @@ -635,7 +657,7 @@ fun MessageBubble( Icon( imageVector = Icons.Filled.Warning, contentDescription = stringResource(R.string.debugging_error), - tint = textColor, + tint = textColor ) Spacer(modifier = Modifier.width(8.dp)) } @@ -653,49 +675,86 @@ fun MessageBubble( val chipBgColor = MaterialTheme.colorScheme.primary val chipTextColor = MaterialTheme.colorScheme.onPrimary val formattedText = - remember(contentText, installedApps, chipTextColor) { - formatMessageText(contentText, installedApps, chipTextColor) + remember(contentText, installedApps) { + formatMessageText(contentText, installedApps) + } + val textMeasurer = rememberTextMeasurer() + val typographyStyle = MaterialTheme.typography.bodyLarge + val density = LocalDensity.current + + val inlineContentMap = + remember( + contentText, + installedApps, + chipBgColor, + chipTextColor, + density + ) { + val map = mutableMapOf() + if (installedApps.isNotEmpty() && contentText.contains("@")) { + val appLabelsPattern = installedApps.joinToString( + "|" + ) { Regex.escape(it.label) } + val regex = + Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + regex.findAll(contentText).forEachIndexed { index, match -> + val id = "chip_$index" + val appName = match.value + val measured = + textMeasurer.measure( + text = appName, + style = typographyStyle.copy( + fontWeight = FontWeight.Bold + ) + ) + val widthSp = + with( + density + ) { (measured.size.width + 8.dp.roundToPx()).toSp() } + val heightSp = + with( + density + ) { (measured.size.height + 2.dp.roundToPx()).toSp() } + + map[id] = + InlineTextContent( + Placeholder( + width = widthSp, + height = heightSp, + placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter + ) + ) { + Surface( + shape = androidx.compose.foundation.shape.RoundedCornerShape( + 6.dp + ), + color = chipBgColor + ) { + Box(contentAlignment = Alignment.Center) { + Text( + text = appName, + color = chipTextColor, + style = typographyStyle.copy( + fontWeight = FontWeight.Bold + ), + modifier = Modifier.padding( + horizontal = 4.dp, + vertical = 1.dp + ) + ) + } + } + } + } + } + map } - var textLayoutResult by remember { mutableStateOf(null) } Text( text = formattedText, + inlineContent = inlineContentMap, color = textColor, - style = MaterialTheme.typography.bodyLarge, - onTextLayout = { textLayoutResult = it }, - modifier = - Modifier.drawBehind { - val layout = textLayoutResult ?: return@drawBehind - formattedText.getStringAnnotations( - tag = "mention", - start = 0, - end = formattedText.length, - ) - .forEach { annotation -> - val start = annotation.start - val end = annotation.end - val path = layout.getPathForRange(start, end) - val rect = path.getBounds() - - val paddingPx = 4.dp.toPx() - val cornerRadiusPx = 6.dp.toPx() - - drawRoundRect( - color = chipBgColor, - topLeft = Offset(rect.left - paddingPx, rect.top), - size = - Size( - rect.width + (paddingPx * 2), - rect.height, - ), - cornerRadius = - CornerRadius( - cornerRadiusPx, - cornerRadiusPx, - ), - ) - } - }, + style = typographyStyle ) } } @@ -707,17 +766,17 @@ fun MessageBubble( enabled = isValidAction, shape = CircleShape, colors = - ButtonDefaults.buttonColors( - containerColor = MaterialTheme.colorScheme.primary, - contentColor = MaterialTheme.colorScheme.onPrimary, - ), + ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary + ) ) { Text( if (isValidAction) { stringResource(R.string.agent_demo_confirm_action) } else { stringResource(R.string.agent_demo_action_expired) - }, + } ) } } @@ -727,24 +786,21 @@ fun MessageBubble( } @Composable -fun StatusIndicator( - status: AgentStatus, - packageManager: PackageManager, -) { +fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { when (status) { AgentStatus.Thinking -> { Row( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically, + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically ) { CircularProgressIndicator(modifier = Modifier.size(24.dp)) Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_thinking), - style = MaterialTheme.typography.bodyMedium, + style = MaterialTheme.typography.bodyMedium ) } } @@ -766,22 +822,22 @@ fun StatusIndicator( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), shape = MaterialTheme.shapes.large, color = MaterialTheme.colorScheme.surfaceBright, - shadowElevation = 2.dp, + shadowElevation = 2.dp ) { Row( modifier = Modifier.padding(12.dp), - verticalAlignment = Alignment.CenterVertically, + verticalAlignment = Alignment.CenterVertically ) { appIcon?.let { Image( bitmap = it.toBitmap().asImageBitmap(), contentDescription = null, - modifier = Modifier.size(40.dp), + modifier = Modifier.size(40.dp) ) Spacer(modifier = Modifier.width(12.dp)) } @@ -792,7 +848,7 @@ fun StatusIndicator( Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_connecting), - style = MaterialTheme.typography.bodyMedium, + style = MaterialTheme.typography.bodyMedium ) } } @@ -811,26 +867,26 @@ fun ChatHistorySidePanel( threads: List, currentThread: ThreadEntity?, onEvent: (AgentUiEvent) -> Unit, - modifier: Modifier = Modifier, + modifier: Modifier = Modifier ) { Column( modifier = - modifier - .width(280.dp) - .fillMaxHeight() - .padding(16.dp), + modifier + .width(280.dp) + .fillMaxHeight() + .padding(16.dp) ) { Text( text = stringResource(R.string.agent_demo_chat_history), style = MaterialTheme.typography.titleLarge, color = MaterialTheme.colorScheme.onSurface, - modifier = Modifier.padding(bottom = 16.dp), + modifier = Modifier.padding(bottom = 16.dp) ) LazyColumn(modifier = Modifier.fillMaxSize()) { items( items = threads, - key = { thread -> thread.threadId }, + key = { thread -> thread.threadId } ) { thread -> val isSelected = thread.threadId == currentThread?.threadId val backgroundColor = @@ -848,26 +904,26 @@ fun ChatHistorySidePanel( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp) - .clickable { - onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) - }, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp) + .clickable { + onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) + }, shape = MaterialTheme.shapes.medium, color = backgroundColor, - contentColor = textColor, + contentColor = textColor ) { Column(modifier = Modifier.padding(12.dp)) { Text( text = thread.llmModel.modelName, style = MaterialTheme.typography.bodyMedium, - color = textColor, + color = textColor ) Text( text = "ID: ${thread.threadId.take(8)}", style = MaterialTheme.typography.bodySmall, - color = textColor.copy(alpha = 0.7f), + color = textColor.copy(alpha = 0.7f) ) } } @@ -878,7 +934,7 @@ fun ChatHistorySidePanel( class InlineAppScopingVisualTransformation( private val installedApps: List, - private val chipTextColor: Color, + private val chipTextColor: Color ) : VisualTransformation { private val regex: Regex? = if (installedApps.isNotEmpty()) { @@ -906,8 +962,8 @@ class InlineAppScopingVisualTransformation( withStyle( SpanStyle( color = chipTextColor, - fontWeight = FontWeight.Bold, - ), + fontWeight = FontWeight.Bold + ) ) { append(match.value) } @@ -922,11 +978,7 @@ class InlineAppScopingVisualTransformation( } } -fun formatMessageText( - text: String, - installedApps: List, - chipTextColor: Color, -): AnnotatedString { +fun formatMessageText(text: String, installedApps: List): AnnotatedString { if (installedApps.isEmpty() || !text.contains("@")) { return AnnotatedString(text) } @@ -936,37 +988,12 @@ fun formatMessageText( return buildAnnotatedString { var lastIndex = 0 - matches.forEach { match -> + matches.forEachIndexed { index, match -> val precedingText = text.substring(lastIndex, match.range.first) if (precedingText.isNotEmpty()) { append(precedingText) } - - pushStringAnnotation(tag = "mention", annotation = match.value) - val appName = match.value - if (appName.isNotEmpty()) { - val mainPart = appName.dropLast(1) - val lastChar = appName.takeLast(1) - - withStyle( - SpanStyle( - color = chipTextColor, - fontWeight = FontWeight.Bold, - ), - ) { - append(mainPart) - } - withStyle( - SpanStyle( - color = chipTextColor, - fontWeight = FontWeight.Bold, - letterSpacing = 3.sp, - ), - ) { - append(lastChar) - } - } - pop() + appendInlineContent(id = "chip_$index", alternateText = match.value) lastIndex = match.range.last + 1 } if (lastIndex < text.length) { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index 06ba145..0e7a34a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -27,6 +27,9 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.appfunction.AppInfo +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -34,6 +37,8 @@ import com.example.appfunctions.agent.domain.pendingintent.ConsumePendingIntentU import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase import com.example.appfunctions.agent.domain.pendingintent.ObserveActivePendingIntentsUseCase import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -42,179 +47,178 @@ import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch -import javax.inject.Inject - -import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase -import com.example.appfunctions.agent.domain.appfunction.AppInfo -import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import kotlinx.coroutines.withContext @HiltViewModel class AgentDemoViewModel - @Inject - constructor( - private val savedStateHandle: SavedStateHandle, - private val getChatHistoryUseCase: GetChatHistoryUseCase, - private val manageThreadsUseCase: ManageThreadsUseCase, - private val sendMessageUseCase: SendMessageUseCase, - private val agentOrchestrator: AgentOrchestrator, - private val settingsRepository: SettingsRepository, - private val observeActivePendingIntentsUseCase: ObserveActivePendingIntentsUseCase, - private val launchPendingIntentUseCase: LaunchPendingIntentUseCase, - private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, - private val getInstalledAppsUseCase: GetInstalledAppsUseCase, - private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - ) : ViewModel() { - private val _uiState = MutableStateFlow(AgentUiState.Loading) - val uiState: StateFlow = _uiState.asStateFlow() +@Inject +constructor( + private val savedStateHandle: SavedStateHandle, + private val getChatHistoryUseCase: GetChatHistoryUseCase, + private val manageThreadsUseCase: ManageThreadsUseCase, + private val sendMessageUseCase: SendMessageUseCase, + private val agentOrchestrator: AgentOrchestrator, + private val settingsRepository: SettingsRepository, + private val observeActivePendingIntentsUseCase: ObserveActivePendingIntentsUseCase, + private val launchPendingIntentUseCase: LaunchPendingIntentUseCase, + private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, + private val getInstalledAppsUseCase: GetInstalledAppsUseCase, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase +) : ViewModel() { + private val _uiState = MutableStateFlow(AgentUiState.Loading) + val uiState: StateFlow = _uiState.asStateFlow() - private val _installedApps = MutableStateFlow>(emptyList()) + private val _installedApps = MutableStateFlow>(emptyList()) - private var observationJob: Job? = null + private var observationJob: Job? = null - init { - viewModelScope.launch { - getAppFunctionsUseCase().collect { toolsMap -> - val allTools = toolsMap.values.flatten() - val packagesWithTools = allTools.map { it.packageName }.toSet() - _installedApps.value = getInstalledAppsUseCase().filter { it.packageName in packagesWithTools } + init { + viewModelScope.launch { + getAppFunctionsUseCase().collect { toolsMap -> + val allTools = toolsMap.values.flatten() + val packagesWithTools = allTools.map { it.packageName }.toSet() + val filteredApps = withContext(Dispatchers.IO) { + getInstalledAppsUseCase().filter { it.packageName in packagesWithTools } } + _installedApps.value = filteredApps } + } - viewModelScope.launch { - observeActivePendingIntentsUseCase().collect { activePendingActionIds -> - val currentState = _uiState.value - if (currentState is AgentUiState.Loaded) { - _uiState.value = - currentState.copy(activePendingActionIds = activePendingActionIds) - } + viewModelScope.launch { + observeActivePendingIntentsUseCase().collect { activePendingActionIds -> + val currentState = _uiState.value + if (currentState is AgentUiState.Loaded) { + _uiState.value = + currentState.copy(activePendingActionIds = activePendingActionIds) } } + } - viewModelScope.launch { - val threads = manageThreadsUseCase.getThreads().first() - if (threads.isEmpty()) { - createAndSelectThread(LlmModel.DEFAULT) - } + viewModelScope.launch { + val threads = manageThreadsUseCase.getThreads().first() + if (threads.isEmpty()) { + createAndSelectThread(LlmModel.DEFAULT) } + } - viewModelScope.launch { - combine( - manageThreadsUseCase.getThreads(), - settingsRepository.selectedProvider, - agentOrchestrator.status, - savedStateHandle.getStateFlow(MainActivity.ARG_THREAD_ID, null), - _installedApps, - ) { - threads, - provider, - status, - targetThreadId, - apps, - -> - ThreadConfig(threads, provider, status, targetThreadId, apps) - } - .collectLatest { (threads, provider, status, targetThreadId, apps) -> - val currentThread = - threads.find { it.threadId == targetThreadId } ?: threads.firstOrNull() + viewModelScope.launch { + combine( + manageThreadsUseCase.getThreads(), + settingsRepository.selectedProvider, + agentOrchestrator.status, + savedStateHandle.getStateFlow(MainActivity.ARG_THREAD_ID, null), + _installedApps + ) { + threads, + provider, + status, + targetThreadId, + apps + -> + ThreadConfig(threads, provider, status, targetThreadId, apps) + } + .collectLatest { (threads, provider, status, targetThreadId, apps) -> + val currentThread = + threads.find { it.threadId == targetThreadId } ?: threads.firstOrNull() - val previousThreadId = - (_uiState.value as? AgentUiState.Loaded)?.currentThread?.threadId + val previousThreadId = + (_uiState.value as? AgentUiState.Loaded)?.currentThread?.threadId - if (currentThread == null) { - observationJob?.cancel() - observationJob = null - _uiState.value = AgentUiState.Loading - } else { - val currentLoadedState = _uiState.value as? AgentUiState.Loaded - _uiState.value = - AgentUiState.Loaded( - currentThread = currentThread, - messages = currentLoadedState?.messages ?: emptyList(), - status = status, - threads = threads, - activePendingActionIds = - currentLoadedState?.activePendingActionIds ?: emptySet(), - installedApps = apps, - ) + if (currentThread == null) { + observationJob?.cancel() + observationJob = null + _uiState.value = AgentUiState.Loading + } else { + val currentLoadedState = _uiState.value as? AgentUiState.Loaded + _uiState.value = + AgentUiState.Loaded( + currentThread = currentThread, + messages = currentLoadedState?.messages ?: emptyList(), + status = status, + threads = threads, + activePendingActionIds = + currentLoadedState?.activePendingActionIds ?: emptySet(), + installedApps = apps + ) - // Start observing messages for the current thread if not already doing so - if (observationJob == null || previousThreadId != currentThread.threadId) { - observationJob?.cancel() - observationJob = - viewModelScope.launch { - launch { - getChatHistoryUseCase(currentThread.threadId).collect { - messages -> - val currentState = _uiState.value - if (currentState is AgentUiState.Loaded) { - _uiState.value = - currentState.copy(messages = messages) - } + // Start observing messages for the current thread if not already doing so + if (observationJob == null || previousThreadId != currentThread.threadId) { + observationJob?.cancel() + observationJob = + viewModelScope.launch { + launch { + getChatHistoryUseCase(currentThread.threadId).collect { + messages -> + val currentState = _uiState.value + if (currentState is AgentUiState.Loaded) { + _uiState.value = + currentState.copy(messages = messages) } } - launch { - agentOrchestrator.observeAndProcessMessages( - currentThread.threadId, - ) - } } - } + launch { + agentOrchestrator.observeAndProcessMessages( + currentThread.threadId + ) + } + } } } - } + } } + } - fun onEvent(event: AgentUiEvent) { - val currentState = _uiState.value - when (event) { - is AgentUiEvent.OnSendMessage -> { - if (currentState is AgentUiState.Loaded) { - viewModelScope.launch { - sendMessageUseCase( - threadId = currentState.currentThread.threadId, - role = MessageRole.USER, - textContent = event.text, - processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - targetPackageName = event.targetPackageName, - ) - } + fun onEvent(event: AgentUiEvent) { + val currentState = _uiState.value + when (event) { + is AgentUiEvent.OnSendMessage -> { + if (currentState is AgentUiState.Loaded) { + viewModelScope.launch { + sendMessageUseCase( + threadId = currentState.currentThread.threadId, + role = MessageRole.USER, + textContent = event.text, + processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, + targetPackageName = event.targetPackageName + ) } } - is AgentUiEvent.OnModelSelected -> { - if (currentState is AgentUiState.Loaded) { - viewModelScope.launch { - manageThreadsUseCase.updateThreadModel( - currentState.currentThread.threadId, - event.model, - ) - } + } + is AgentUiEvent.OnModelSelected -> { + if (currentState is AgentUiState.Loaded) { + viewModelScope.launch { + manageThreadsUseCase.updateThreadModel( + currentState.currentThread.threadId, + event.model + ) } } - is AgentUiEvent.OnCreateThread -> { - viewModelScope.launch { createAndSelectThread(event.model) } - } - is AgentUiEvent.OnThreadSelected -> { - savedStateHandle[MainActivity.ARG_THREAD_ID] = event.threadId - } - is AgentUiEvent.OnConfirmAction -> { - val pendingIntent = consumePendingIntentUseCase(event.pendingIntentId) - if (pendingIntent != null) { - launchPendingIntentUseCase(pendingIntent) - } + } + is AgentUiEvent.OnCreateThread -> { + viewModelScope.launch { createAndSelectThread(event.model) } + } + is AgentUiEvent.OnThreadSelected -> { + savedStateHandle[MainActivity.ARG_THREAD_ID] = event.threadId + } + is AgentUiEvent.OnConfirmAction -> { + val pendingIntent = consumePendingIntentUseCase(event.pendingIntentId) + if (pendingIntent != null) { + launchPendingIntentUseCase(pendingIntent) } } } + } - private suspend fun createAndSelectThread(llmModel: LlmModel) { - val threadId = manageThreadsUseCase.createThread(llmModel) - savedStateHandle[MainActivity.ARG_THREAD_ID] = threadId - } + private suspend fun createAndSelectThread(llmModel: LlmModel) { + val threadId = manageThreadsUseCase.createThread(llmModel) + savedStateHandle[MainActivity.ARG_THREAD_ID] = threadId } +} private data class ThreadConfig( val threads: List, val provider: LlmProviderName, val status: AgentStatus, val targetThreadId: String?, - val installedApps: List, + val installedApps: List )