Voice Assistant Widget

dependencies {
    implementation("ai.liquid.leap:leap-sdk:0.10.6")
    implementation("ai.liquid.leap:leap-ui:0.10.6")
}

import ai.liquid.leap.model_downloader.LeapModelDownloader
import ai.liquid.leap.ui.VoiceAssistantIntent
import ai.liquid.leap.ui.VoiceAssistantStore
import ai.liquid.leap.ui.VoiceAssistantStoreState
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch

class VoiceAssistantViewModel(application: Application) : AndroidViewModel(application) {
    private val recorder = AndroidAudioRecorder()    // see "Audio I/O implementations" below
    private val player = AndroidAudioPlayer()

    val store = VoiceAssistantStore(recorder = recorder, player = player, scope = viewModelScope)
    val state: StateFlow<VoiceAssistantStoreState> = store.state

    private val downloader = LeapModelDownloader(application)

    init { viewModelScope.launch { loadModel() } }

    fun processIntent(intent: VoiceAssistantIntent) = store.processIntent(intent)

    private suspend fun loadModel() = runCatching {
        store.setModelProgress(0f, "Resolving manifest…")
        val runner = downloader.loadModel(
            modelName = "LFM2.5-Audio-1.5B",
            quantizationType = "Q4_0",
            progress = { pd ->
                val pct = if (pd.total > 0) " (${(pd.bytes * 100 / pd.total).toInt()}%)" else ""
                store.setModelProgress(
                    fraction = if (pd.total > 0) pd.bytes.toFloat() / pd.total else 0f,
                    message = "Downloading$pct",
                )
            },
        )
        store.setConversation(
            LeapVoiceConversation(
                conv = runner.createConversation(systemPrompt = "Respond with interleaved text and audio.")
            )
        )
    }.onFailure { e -> store.setModelError("✗ ${e.message}") }

    override fun onCleared() {
        super.onCleared()
        store.close()
    }
}

import LeapUi
import SwiftUI

private struct VoiceWidgetRepresentable: NSViewControllerRepresentable {
    let store: VoiceAssistantStore

    func makeNSViewController(context: Context) -> NSViewController {
        VoiceAssistantNSViewControllerKt.VoiceAssistantNSViewController(
            state: store.widgetStateHolder,
            onIntent: { intent in store.processIntent(intent: intent) },
            labels: VoiceWidgetLabels(/* same labels */),
            colors: VoiceWidgetColors.companion.Default,
            showPoweredBy: true
        )
    }

    func updateNSViewController(_ nsViewController: NSViewController, context: Context) {}
}

import ai.liquid.leap.ui.VoiceAssistantWidget
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.darkColorScheme
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.lifecycle.viewmodel.compose.viewModel

class MainActivity : ComponentActivity() {
    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContent {
            MaterialTheme(colorScheme = darkColorScheme(background = Color.Black)) {
                val vm = viewModel<VoiceAssistantViewModel>()
                val state by vm.state.collectAsState()

                VoiceAssistantWidget(
                    state = state.widgetState,
                    onIntent = vm::processIntent,
                    modifier = Modifier.fillMaxSize().background(Color.Black),
                )
            }
        }
    }
}

import ai.liquid.leap.Conversation
import ai.liquid.leap.MessageResponse
import ai.liquid.leap.message.ChatMessage
import ai.liquid.leap.message.ChatMessageContent
import ai.liquid.leap.message.GenerationStats
import ai.liquid.leap.message.encodePcm16Wav
import ai.liquid.leap.ui.VoiceConversation

class LeapVoiceConversation(private val conv: Conversation) : VoiceConversation {

    override suspend fun generateResponse(
        audioSamples: FloatArray,
        sampleRate: Int,
        onAudioChunk: (samples: FloatArray, sampleRate: Int) -> Unit,
    ): GenerationStats? {
        val wavBytes = encodePcm16Wav(audioSamples, sampleRate)
        val userMessage = ChatMessage(
            role = ChatMessage.Role.USER,
            content = listOf(ChatMessageContent.Audio(wavBytes)),
        )

        var stats: GenerationStats? = null
        conv.generateResponse(userMessage).collect { response ->
            when (response) {
                is MessageResponse.AudioSample -> onAudioChunk(response.samples, response.sampleRate)
                is MessageResponse.Complete -> stats = response.stats
                else -> Unit
            }
        }
        return stats
    }

    override fun reset(): VoiceConversation =
        LeapVoiceConversation(conv.modelRunner.createConversation())
}

<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.INTERNET" />

val store = VoiceAssistantStore(
    recorder = recorder,
    player = player,
    scope = viewModelScope,
    interruptToSpeak = false,
)