Files
WhisperClient2/app/src/main/java/com/objectbrokers/whisperclient2/MainActivity.kt
T

633 lines
29 KiB
Kotlin

package com.objectbrokers.whisperclient2
import android.Manifest
import android.annotation.SuppressLint
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.os.Bundle
import android.util.Log
import android.widget.Toast
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.Button
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.lifecycle.lifecycleScope
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.Job
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import okhttp3.WebSocket
import okio.ByteString
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocketListener
import com.objectbrokers.whisperclient2.ui.theme.WhisperClient2Theme
import kotlinx.coroutines.isActive
import kotlinx.coroutines.withContext
import okio.ByteString.Companion.toByteString
import org.json.JSONObject
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.UUID
import java.util.concurrent.TimeUnit
import kotlin.math.roundToInt
const val TAG = "SpeechClient"
class MainActivity : ComponentActivity() {
private var recordingJob: Job? = null
private var audioCaptureJob: Job? = null
// private var isActive: Boolean = false
private lateinit var audioRecord: AudioRecord
private var webSocket: WebSocket? = null
private var clientUid: UUID? = null
private var bufferSize: Int = 0
private var buffer: ByteArray? = null
private var isRecording by mutableStateOf(false)
private var transcriptText by mutableStateOf("Transcription will appear here...")
private var connectionStatus by mutableStateOf("Disconnected")
private lateinit var settingsManager: SettingsManager
private val client = OkHttpClient.Builder()
.pingInterval(30, TimeUnit.SECONDS)
.build()
private val requestPermissionLauncher = registerForActivityResult(
ActivityResultContracts.RequestPermission()
) { isGranted: Boolean ->
if (isGranted) {
// Permission granted! You can now call startRecording()
} else {
// Permission denied. Show a message to the user.
}
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
settingsManager = SettingsManager(this)
enableEdgeToEdge()
setContent {
val settings by settingsManager.settingsFlow.collectAsState(
initial = HandshakeSettings(
SettingsManager.DEFAULT_HOST, SettingsManager.DEFAULT_PORT,
SettingsManager.DEFAULT_TASK, SettingsManager.DEFAULT_MODEL,
SettingsManager.DEFAULT_LANGUAGE, SettingsManager.DEFAULT_USE_VAD,
SettingsManager.DEFAULT_AUDIO_FORMAT, SettingsManager.DEFAULT_SEND_LAST_N_SEGMENTS,
SettingsManager.DEFAULT_NO_SPEECH_THRESH, SettingsManager.DEFAULT_CLIP_AUDIO,
SettingsManager.DEFAULT_SAME_OUTPUT_THRESHOLD, SettingsManager.DEFAULT_ENABLE_TRANSLATION,
SettingsManager.DEFAULT_TARGET_LANGUAGE, null,
SettingsManager.DEFAULT_ENABLE_DIARIZATION, SettingsManager.DEFAULT_MAX_SPEAKERS,
SettingsManager.DEFAULT_WORD_TIMESTAMPS
)
)
var showSettings by remember { mutableStateOf(false) }
WhisperClient2Theme {
WhisperApp(
transcriptText = transcriptText,
isRecording = isRecording,
connectionStatus = connectionStatus,
onOpenSettings = { showSettings = true },
onToggleRecording = {
if (isRecording) {
stopRecording()
isRecording = false
} else {
startRecording()
isRecording = true
}
}
)
if (showSettings) {
SettingsDialog(
initialSettings = settings,
onDismiss = { showSettings = false },
onSave = { newSettings ->
lifecycleScope.launch {
settingsManager.saveSettings(newSettings)
showSettings = false
}
}
)
}
}
}
when {
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED -> {
// Already have permission
}
else -> {
// Ask for it
requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
}
}
}
class AudioInitializationException(message: String) : Exception(message)
@SuppressLint("MissingPermission") // Ensure runtime check is completed beforehand
fun createAudioRecorder(): AudioRecord {
// 1. Get the minimum buffer size required for the hardware configuration
val minBufferSize = AudioRecord.getMinBufferSize(
16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)
// 2. Build and return the AudioRecord instance
var recorder: AudioRecord? = null;
recorder = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(16000)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
// Double or triple the min buffer size to prevent buffer overflow (over-runs)
.setBufferSizeInBytes(minBufferSize * 2)
.build()
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
recorder.release()
throw AudioInitializationException("AudioRecord failed to initialize — mic unavailable or permission denied")
}
return recorder
}
fun startRecording() {
recordingJob = lifecycleScope.launch(Dispatchers.IO) {
var audioRecordOK = true
bufferSize = AudioRecord.getMinBufferSize(
16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
) * 2
buffer = ByteArray(bufferSize)
// Create the AudioRecord instance
try {
audioRecord = createAudioRecorder()
} catch (e: AudioInitializationException) {
Log.e(TAG, "Audio init failed: ${e.message}")
withContext(Dispatchers.Main) {
Toast.makeText(this@MainActivity, "Audio init failed: ${e.message}", Toast.LENGTH_LONG).show()
audioRecordOK = false
}
}
if (audioRecordOK) {
withContext(Dispatchers.Main) {
connectionStatus = "Connecting"
}
// Create the WebSocket instance
val settings = settingsManager.settingsFlow.first()
val request = Request.Builder()
.url("ws://${settings.host}:${settings.port}")
.build()
val listener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
// get a new client UUID (unique session ID) for each connection
clientUid = UUID.randomUUID()
// Build the handshake JSON using Android's built-in JSONObject.
val handshake = JSONObject().apply {
put("uid", clientUid.toString())
put("task", settings.task)
put("model", settings.model)
put("language", settings.language)
put("use_vad", settings.useVad)
put("audio_format", settings.audioFormat)
put("send_last_n_segments", settings.sendLastNSegments)
put("no_speech_thresh", settings.noSpeechThresh)
put("clip_audio", settings.clipAudio)
put("same_output_threshold", settings.sameOutputThreshold)
put("enable_translation", settings.enableTranslation)
put("target_language", settings.targetLanguage)
put("hotwords", settings.hotwords ?: JSONObject.NULL)
put("enable_diarization", settings.enableDiarization)
put("max_speakers", settings.maxSpeakers)
put("word_timestamps", settings.wordTimestamps)
}
// send() with a String argument sends a WebSocket text frame
webSocket.send(handshake.toString())
}
// Handle onClosing, onFailure, etc.
override fun onMessage(webSocket: WebSocket, text: String) {
Log.i(TAG, "Received message: $text")
val message = JSONObject(text)
// 1. Validate this message is meant for us.
// Store the uid you sent in the handshake as a class member.
if (message.optString("uid") != clientUid.toString()) {
Log.e(TAG, "Received message with wrong uid — ignoring")
return
}
// 2. Status messages — server full, error, or warning.
// These arrive at any time, not just after the handshake.
if (message.has("status")) {
when (message.getString("status")) {
"WAIT" -> {
val minutes = message.getDouble("message").roundToInt()
Log.i(TAG, "Server full. Estimated wait: $minutes minutes")
runOnUiThread {
connectionStatus = "Wait ($minutes min)"
}
}
"ERROR" -> {
Log.e(TAG, "Server error: ${message.getString("message")}")
runOnUiThread {
connectionStatus = "Error"
}
}
"WARNING" -> {
Log.w(TAG, "Server warning: ${message.getString("message")}")
}
}
return
}
// 3. Control messages — check the "message" key for named signals.
if (message.has("message")) {
when (message.getString("message")) {
"SERVER_READY" -> {
// This is the handshake acknowledgement you asked about.
// Only start sending audio after receiving this.
val backend =
message.getString("backend") // e.g. "faster_whisper"
Log.i(TAG, "Server ready, backend: $backend")
runOnUiThread {
connectionStatus = "Server Ready"
}
startAudioCapture() // <-- safe to start sending audio now
}
"DISCONNECT" -> {
// Server hung up because the session exceeded MAX_CONNECTION_TIME
Log.i(TAG, "Server disconnected: session time limit reached")
stopRecording()
}
}
return
}
// 4. Language detection — server tells us what language it detected.
// Only relevant if you sent language=null in the handshake.
if (message.has("language")) {
val lang = message.getString("language")
val prob = message.getDouble("language_prob")
Log.i(TAG, "Detected language: $lang (probability: $prob)")
return
}
// 5. Transcript segments — the normal steady-state message during recording.
if (message.has("segments")) {
val segments = message.getJSONArray("segments")
val sb = StringBuilder()
for (i in 0 until segments.length()) {
sb.append(segments.getJSONObject(i).getString("text").trim())
if (i < segments.length() - 1) sb.append(" ")
}
Log.i(TAG, "Transcript: $sb")
runOnUiThread {
val text12: String = sb.toString()
this@MainActivity.transcriptText = text12
// transcriptTextView.text = transcriptText
}
}
}
override fun onClosing(webSocket: WebSocket, code: Int, reason: String) {
Log.e(TAG, "WebSocket closing: $code / $reason")
runOnUiThread {
if (code != 1000) {
connectionStatus = "Error"
}
}
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
Log.i(TAG, "WebSocket closed: $code / $reason")
runOnUiThread {
if (connectionStatus != "Error") {
connectionStatus = "Disconnected"
}
}
}
override fun onFailure(
webSocket: WebSocket,
t: Throwable,
response: Response?
) {
Log.e(TAG, "WebSocket connection failed: ${t.message}", t)
runOnUiThread {
connectionStatus = "Error"
}
}
}
webSocket = client.newWebSocket(request, listener)
Log.i(TAG, "WebSocket: $webSocket")
}
}
}
fun stopRecording() {
connectionStatus = "Disconnected"
stopAudioCapture()
recordingJob?.cancel() // the ?. is Kotlin null-safety — no NPE if job is null
webSocket?.close(1000, "Done")
webSocket = null
}
fun stopAudioCapture() {
audioCaptureJob?.cancel()
audioRecord.stop()
audioRecord.release()
}
fun startAudioCapture() {
audioCaptureJob = lifecycleScope.launch(Dispatchers.IO) {
audioRecord.startRecording();
Log.i(TAG, "audioRecord.state: ${audioRecord.state}")
while (isActive) {
buffer?.let {
val bytesRead = audioRecord.read(it, 0, bufferSize)
Log.i(TAG, "bytesRead: $bytesRead")
if (bytesRead > 0) {
// Check RMS energy — if this is always near zero, AudioRecord
// is returning silence regardless of what the mic hears
val shorts = ShortArray(bytesRead / 2)
ByteBuffer.wrap(it, 0, bytesRead)
.order(ByteOrder.LITTLE_ENDIAN)
.asShortBuffer()
.get(shorts)
val rms = Math.sqrt(shorts.map { s -> s.toDouble() * s }.average())
Log.d(TAG, "Audio chunk RMS: $rms")
webSocket?.send(it.toByteString(0, bytesRead))
}
}
}
}
}
fun sendAudioPacket(webSocket: WebSocket, audioBytes: ByteArray) {
// OkHttp's send() is overloaded:
// send(String) → text frame
// send(ByteString) → binary frame ← this is what we want
// ByteString.of() wraps a ByteArray without copying if possible.
webSocket.send(ByteString.of(*audioBytes))
}
}
/*
@Composable
fun WhisperApp(text: String)
{ // ... inside your scrollable column ... Text( text = text,
// It is now tied to the 'transcriptText' variable style = MaterialTheme.typography.bodyLarge ) }
*/
@OptIn(ExperimentalMaterial3Api::class)
@Composable
fun WhisperApp(
transcriptText: String,
isRecording: Boolean = false,
connectionStatus: String = "Disconnected",
onOpenSettings: () -> Unit = {},
onToggleRecording: () -> Unit = {}
) {
Scaffold(
modifier = Modifier.fillMaxSize(),
topBar = {
TopAppBar(
title = { Text("Whisper Client") },
actions = {
IconButton(onClick = onOpenSettings) {
Icon(Icons.Default.Settings, contentDescription = "Settings")
}
}
)
}
) { innerPadding ->
Column(
modifier = Modifier
.padding(innerPadding)
.fillMaxSize()
.padding(16.dp),
horizontalAlignment = Alignment.CenterHorizontally
) {
Text(
text = "Transcription",
style = MaterialTheme.typography.headlineMedium,
fontWeight = FontWeight.Bold,
modifier = Modifier.padding(bottom = 16.dp)
)
Column(
modifier = Modifier
.weight(1f)
.fillMaxWidth()
.verticalScroll(rememberScrollState())
) {
Text(
text = transcriptText,
style = MaterialTheme.typography.bodyLarge
)
}
Row(
modifier = Modifier
.fillMaxWidth()
.padding(top = 16.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween
) {
Button(
onClick = onToggleRecording,
modifier = Modifier.weight(1f),
content = { Text(if (isRecording) "Stop Transcription" else "Start Transcription") }
)
Spacer(modifier = Modifier.width(16.dp))
Text(
text = connectionStatus,
style = MaterialTheme.typography.bodyMedium,
fontWeight = FontWeight.Bold,
color = when (connectionStatus) {
"Server Ready" -> Color(0xFF4CAF50) // Green
"Error" -> MaterialTheme.colorScheme.error
"Connecting" -> Color(0xFFFFA000) // Amber
else -> MaterialTheme.colorScheme.onSurfaceVariant
}
)
}
}
}
}
@Composable
fun SettingsDialog(
initialSettings: HandshakeSettings,
onDismiss: () -> Unit,
onSave: (HandshakeSettings) -> Unit
) {
var host by remember { mutableStateOf(initialSettings.host) }
var port by remember { mutableStateOf(initialSettings.port) }
var task by remember { mutableStateOf(initialSettings.task) }
var model by remember { mutableStateOf(initialSettings.model) }
var language by remember { mutableStateOf(initialSettings.language) }
var useVad by remember { mutableStateOf(initialSettings.useVad) }
var audioFormat by remember { mutableStateOf(initialSettings.audioFormat) }
var sendLastNSegments by remember { mutableStateOf(initialSettings.sendLastNSegments.toString()) }
var noSpeechThresh by remember { mutableStateOf(initialSettings.noSpeechThresh.toString()) }
var clipAudio by remember { mutableStateOf(initialSettings.clipAudio) }
var sameOutputThreshold by remember { mutableStateOf(initialSettings.sameOutputThreshold.toString()) }
var enableTranslation by remember { mutableStateOf(initialSettings.enableTranslation) }
var targetLanguage by remember { mutableStateOf(initialSettings.targetLanguage) }
var hotwords by remember { mutableStateOf(initialSettings.hotwords ?: "") }
var enableDiarization by remember { mutableStateOf(initialSettings.enableDiarization) }
var maxSpeakers by remember { mutableStateOf(initialSettings.maxSpeakers.toString()) }
var wordTimestamps by remember { mutableStateOf(initialSettings.wordTimestamps) }
AlertDialog(
onDismissRequest = onDismiss,
title = { Text("Server & Handshake Settings") },
text = {
Column(modifier = Modifier.verticalScroll(rememberScrollState())) {
Text("Network", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold)
OutlinedTextField(value = host, onValueChange = { host = it }, label = { Text("Host / IP") }, modifier = Modifier.fillMaxWidth())
OutlinedTextField(value = port, onValueChange = { port = it }, label = { Text("Port") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp))
Text("Whisper Configuration", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold)
OutlinedTextField(value = task, onValueChange = { task = it }, label = { Text("Task") }, modifier = Modifier.fillMaxWidth())
OutlinedTextField(value = model, onValueChange = { model = it }, label = { Text("Model") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
OutlinedTextField(value = language, onValueChange = { language = it }, label = { Text("Language") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
OutlinedTextField(value = audioFormat, onValueChange = { audioFormat = it }, label = { Text("Audio Format") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
OutlinedTextField(value = hotwords, onValueChange = { hotwords = it }, label = { Text("Hotwords (Optional)") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) {
Text("Use VAD", modifier = Modifier.weight(1f))
Switch(checked = useVad, onCheckedChange = { useVad = it })
}
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) {
Text("Clip Audio", modifier = Modifier.weight(1f))
Switch(checked = clipAudio, onCheckedChange = { clipAudio = it })
}
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) {
Text("Word Timestamps", modifier = Modifier.weight(1f))
Switch(checked = wordTimestamps, onCheckedChange = { wordTimestamps = it })
}
HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp))
Text("Thresholds & Segments", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold)
OutlinedTextField(value = sendLastNSegments, onValueChange = { sendLastNSegments = it }, label = { Text("Send Last N Segments") }, modifier = Modifier.fillMaxWidth())
OutlinedTextField(value = noSpeechThresh, onValueChange = { noSpeechThresh = it }, label = { Text("No Speech Threshold") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
OutlinedTextField(value = sameOutputThreshold, onValueChange = { sameOutputThreshold = it }, label = { Text("Same Output Threshold") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp))
Text("Translation", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold)
Row(verticalAlignment = Alignment.CenterVertically) {
Text("Enable Translation", modifier = Modifier.weight(1f))
Switch(checked = enableTranslation, onCheckedChange = { enableTranslation = it })
}
OutlinedTextField(value = targetLanguage, onValueChange = { targetLanguage = it }, label = { Text("Target Language") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp))
Text("Diarization", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold)
Row(verticalAlignment = Alignment.CenterVertically) {
Text("Enable Diarization", modifier = Modifier.weight(1f))
Switch(checked = enableDiarization, onCheckedChange = { enableDiarization = it })
}
OutlinedTextField(value = maxSpeakers, onValueChange = { maxSpeakers = it }, label = { Text("Max Speakers") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp))
}
},
confirmButton = {
TextButton(onClick = {
onSave(
HandshakeSettings(
host = host, port = port, task = task, model = model, language = language,
useVad = useVad, audioFormat = audioFormat,
sendLastNSegments = sendLastNSegments.toIntOrNull() ?: SettingsManager.DEFAULT_SEND_LAST_N_SEGMENTS,
noSpeechThresh = noSpeechThresh.toDoubleOrNull() ?: SettingsManager.DEFAULT_NO_SPEECH_THRESH,
clipAudio = clipAudio,
sameOutputThreshold = sameOutputThreshold.toIntOrNull() ?: SettingsManager.DEFAULT_SAME_OUTPUT_THRESHOLD,
enableTranslation = enableTranslation, targetLanguage = targetLanguage,
hotwords = hotwords.takeIf { it.isNotBlank() },
enableDiarization = enableDiarization,
maxSpeakers = maxSpeakers.toIntOrNull() ?: SettingsManager.DEFAULT_MAX_SPEAKERS,
wordTimestamps = wordTimestamps
)
)
}) {
Text("Save")
}
},
dismissButton = {
TextButton(onClick = onDismiss) {
Text("Cancel")
}
}
)
}
@Preview(showBackground = true)
@Composable
fun WhisperAppPreview() {
WhisperClient2Theme {
WhisperApp(
transcriptText = "Transcription will appear here...",
connectionStatus = "Disconnected"
)
}
}