package com.objectbrokers.whisperclient2 import android.Manifest import android.annotation.SuppressLint import android.content.pm.PackageManager import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaRecorder import android.os.Bundle import android.util.Log import android.widget.Toast import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.width import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Scaffold import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.material3.TopAppBar import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Settings import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat import androidx.lifecycle.lifecycleScope import kotlinx.coroutines.flow.first import kotlinx.coroutines.Job import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import okhttp3.WebSocket import okio.ByteString import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import okhttp3.WebSocketListener import com.objectbrokers.whisperclient2.ui.theme.WhisperClient2Theme import kotlinx.coroutines.isActive import kotlinx.coroutines.withContext import okio.ByteString.Companion.toByteString import org.json.JSONObject import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.UUID import java.util.concurrent.TimeUnit import kotlin.math.roundToInt const val TAG = "SpeechClient" class MainActivity : ComponentActivity() { private var recordingJob: Job? = null private var audioCaptureJob: Job? = null // private var isActive: Boolean = false private lateinit var audioRecord: AudioRecord private var webSocket: WebSocket? = null private var clientUid: UUID? = null private var bufferSize: Int = 0 private var buffer: ByteArray? = null private var isRecording by mutableStateOf(false) private var transcriptText by mutableStateOf("Transcription will appear here...") private var connectionStatus by mutableStateOf("Disconnected") private lateinit var settingsManager: SettingsManager private val client = OkHttpClient.Builder() .pingInterval(30, TimeUnit.SECONDS) .build() private val requestPermissionLauncher = registerForActivityResult( ActivityResultContracts.RequestPermission() ) { isGranted: Boolean -> if (isGranted) { // Permission granted! You can now call startRecording() } else { // Permission denied. Show a message to the user. } } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) settingsManager = SettingsManager(this) enableEdgeToEdge() setContent { val settings by settingsManager.settingsFlow.collectAsState( initial = HandshakeSettings( SettingsManager.DEFAULT_HOST, SettingsManager.DEFAULT_PORT, SettingsManager.DEFAULT_TASK, SettingsManager.DEFAULT_MODEL, SettingsManager.DEFAULT_LANGUAGE, SettingsManager.DEFAULT_USE_VAD, SettingsManager.DEFAULT_AUDIO_FORMAT, SettingsManager.DEFAULT_SEND_LAST_N_SEGMENTS, SettingsManager.DEFAULT_NO_SPEECH_THRESH, SettingsManager.DEFAULT_CLIP_AUDIO, SettingsManager.DEFAULT_SAME_OUTPUT_THRESHOLD, SettingsManager.DEFAULT_ENABLE_TRANSLATION, SettingsManager.DEFAULT_TARGET_LANGUAGE, null, SettingsManager.DEFAULT_ENABLE_DIARIZATION, SettingsManager.DEFAULT_MAX_SPEAKERS, SettingsManager.DEFAULT_WORD_TIMESTAMPS ) ) var showSettings by remember { mutableStateOf(false) } WhisperClient2Theme { WhisperApp( transcriptText = transcriptText, isRecording = isRecording, connectionStatus = connectionStatus, onOpenSettings = { showSettings = true }, onToggleRecording = { if (isRecording) { stopRecording() isRecording = false } else { startRecording() isRecording = true } } ) if (showSettings) { SettingsDialog( initialSettings = settings, onDismiss = { showSettings = false }, onSave = { newSettings -> lifecycleScope.launch { settingsManager.saveSettings(newSettings) showSettings = false } } ) } } } when { ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED -> { // Already have permission } else -> { // Ask for it requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) } } } class AudioInitializationException(message: String) : Exception(message) @SuppressLint("MissingPermission") // Ensure runtime check is completed beforehand fun createAudioRecorder(): AudioRecord { // 1. Get the minimum buffer size required for the hardware configuration val minBufferSize = AudioRecord.getMinBufferSize( 16000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT ) // 2. Build and return the AudioRecord instance var recorder: AudioRecord? = null; recorder = AudioRecord.Builder() .setAudioSource(MediaRecorder.AudioSource.MIC) .setAudioFormat( AudioFormat.Builder() .setEncoding(AudioFormat.ENCODING_PCM_16BIT) .setSampleRate(16000) .setChannelMask(AudioFormat.CHANNEL_IN_MONO) .build() ) // Double or triple the min buffer size to prevent buffer overflow (over-runs) .setBufferSizeInBytes(minBufferSize * 2) .build() if (recorder.state != AudioRecord.STATE_INITIALIZED) { recorder.release() throw AudioInitializationException("AudioRecord failed to initialize — mic unavailable or permission denied") } return recorder } fun startRecording() { recordingJob = lifecycleScope.launch(Dispatchers.IO) { var audioRecordOK = true bufferSize = AudioRecord.getMinBufferSize( 16000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT ) * 2 buffer = ByteArray(bufferSize) // Create the AudioRecord instance try { audioRecord = createAudioRecorder() } catch (e: AudioInitializationException) { Log.e(TAG, "Audio init failed: ${e.message}") withContext(Dispatchers.Main) { Toast.makeText(this@MainActivity, "Audio init failed: ${e.message}", Toast.LENGTH_LONG).show() audioRecordOK = false } } if (audioRecordOK) { withContext(Dispatchers.Main) { connectionStatus = "Connecting" } // Create the WebSocket instance val settings = settingsManager.settingsFlow.first() val request = Request.Builder() .url("ws://${settings.host}:${settings.port}") .build() val listener = object : WebSocketListener() { override fun onOpen(webSocket: WebSocket, response: Response) { // get a new client UUID (unique session ID) for each connection clientUid = UUID.randomUUID() // Build the handshake JSON using Android's built-in JSONObject. val handshake = JSONObject().apply { put("uid", clientUid.toString()) put("task", settings.task) put("model", settings.model) put("language", settings.language) put("use_vad", settings.useVad) put("audio_format", settings.audioFormat) put("send_last_n_segments", settings.sendLastNSegments) put("no_speech_thresh", settings.noSpeechThresh) put("clip_audio", settings.clipAudio) put("same_output_threshold", settings.sameOutputThreshold) put("enable_translation", settings.enableTranslation) put("target_language", settings.targetLanguage) put("hotwords", settings.hotwords ?: JSONObject.NULL) put("enable_diarization", settings.enableDiarization) put("max_speakers", settings.maxSpeakers) put("word_timestamps", settings.wordTimestamps) } // send() with a String argument sends a WebSocket text frame webSocket.send(handshake.toString()) } // Handle onClosing, onFailure, etc. override fun onMessage(webSocket: WebSocket, text: String) { Log.i(TAG, "Received message: $text") val message = JSONObject(text) // 1. Validate this message is meant for us. // Store the uid you sent in the handshake as a class member. if (message.optString("uid") != clientUid.toString()) { Log.e(TAG, "Received message with wrong uid — ignoring") return } // 2. Status messages — server full, error, or warning. // These arrive at any time, not just after the handshake. if (message.has("status")) { when (message.getString("status")) { "WAIT" -> { val minutes = message.getDouble("message").roundToInt() Log.i(TAG, "Server full. Estimated wait: $minutes minutes") runOnUiThread { connectionStatus = "Wait ($minutes min)" } } "ERROR" -> { Log.e(TAG, "Server error: ${message.getString("message")}") runOnUiThread { connectionStatus = "Error" } } "WARNING" -> { Log.w(TAG, "Server warning: ${message.getString("message")}") } } return } // 3. Control messages — check the "message" key for named signals. if (message.has("message")) { when (message.getString("message")) { "SERVER_READY" -> { // This is the handshake acknowledgement you asked about. // Only start sending audio after receiving this. val backend = message.getString("backend") // e.g. "faster_whisper" Log.i(TAG, "Server ready, backend: $backend") runOnUiThread { connectionStatus = "Server Ready" } startAudioCapture() // <-- safe to start sending audio now } "DISCONNECT" -> { // Server hung up because the session exceeded MAX_CONNECTION_TIME Log.i(TAG, "Server disconnected: session time limit reached") stopRecording() } } return } // 4. Language detection — server tells us what language it detected. // Only relevant if you sent language=null in the handshake. if (message.has("language")) { val lang = message.getString("language") val prob = message.getDouble("language_prob") Log.i(TAG, "Detected language: $lang (probability: $prob)") return } // 5. Transcript segments — the normal steady-state message during recording. if (message.has("segments")) { val segments = message.getJSONArray("segments") val sb = StringBuilder() for (i in 0 until segments.length()) { sb.append(segments.getJSONObject(i).getString("text").trim()) if (i < segments.length() - 1) sb.append(" ") } Log.i(TAG, "Transcript: $sb") runOnUiThread { val text12: String = sb.toString() this@MainActivity.transcriptText = text12 // transcriptTextView.text = transcriptText } } } override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { Log.e(TAG, "WebSocket closing: $code / $reason") runOnUiThread { if (code != 1000) { connectionStatus = "Error" } } } override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { Log.i(TAG, "WebSocket closed: $code / $reason") runOnUiThread { if (connectionStatus != "Error") { connectionStatus = "Disconnected" } } } override fun onFailure( webSocket: WebSocket, t: Throwable, response: Response? ) { Log.e(TAG, "WebSocket connection failed: ${t.message}", t) runOnUiThread { connectionStatus = "Error" } } } webSocket = client.newWebSocket(request, listener) Log.i(TAG, "WebSocket: $webSocket") } } } fun stopRecording() { connectionStatus = "Disconnected" stopAudioCapture() recordingJob?.cancel() // the ?. is Kotlin null-safety — no NPE if job is null webSocket?.close(1000, "Done") webSocket = null } fun stopAudioCapture() { audioCaptureJob?.cancel() audioRecord.stop() audioRecord.release() } fun startAudioCapture() { audioCaptureJob = lifecycleScope.launch(Dispatchers.IO) { audioRecord.startRecording(); Log.i(TAG, "audioRecord.state: ${audioRecord.state}") while (isActive) { buffer?.let { val bytesRead = audioRecord.read(it, 0, bufferSize) Log.i(TAG, "bytesRead: $bytesRead") if (bytesRead > 0) { // Check RMS energy — if this is always near zero, AudioRecord // is returning silence regardless of what the mic hears val shorts = ShortArray(bytesRead / 2) ByteBuffer.wrap(it, 0, bytesRead) .order(ByteOrder.LITTLE_ENDIAN) .asShortBuffer() .get(shorts) val rms = Math.sqrt(shorts.map { s -> s.toDouble() * s }.average()) Log.d(TAG, "Audio chunk RMS: $rms") webSocket?.send(it.toByteString(0, bytesRead)) } } } } } fun sendAudioPacket(webSocket: WebSocket, audioBytes: ByteArray) { // OkHttp's send() is overloaded: // send(String) → text frame // send(ByteString) → binary frame ← this is what we want // ByteString.of() wraps a ByteArray without copying if possible. webSocket.send(ByteString.of(*audioBytes)) } } /* @Composable fun WhisperApp(text: String) { // ... inside your scrollable column ... Text( text = text, // It is now tied to the 'transcriptText' variable style = MaterialTheme.typography.bodyLarge ) } */ @OptIn(ExperimentalMaterial3Api::class) @Composable fun WhisperApp( transcriptText: String, isRecording: Boolean = false, connectionStatus: String = "Disconnected", onOpenSettings: () -> Unit = {}, onToggleRecording: () -> Unit = {} ) { Scaffold( modifier = Modifier.fillMaxSize(), topBar = { TopAppBar( title = { Text("Whisper Client") }, actions = { IconButton(onClick = onOpenSettings) { Icon(Icons.Default.Settings, contentDescription = "Settings") } } ) } ) { innerPadding -> Column( modifier = Modifier .padding(innerPadding) .fillMaxSize() .padding(16.dp), horizontalAlignment = Alignment.CenterHorizontally ) { Text( text = "Transcription", style = MaterialTheme.typography.headlineMedium, fontWeight = FontWeight.Bold, modifier = Modifier.padding(bottom = 16.dp) ) Column( modifier = Modifier .weight(1f) .fillMaxWidth() .verticalScroll(rememberScrollState()) ) { Text( text = transcriptText, style = MaterialTheme.typography.bodyLarge ) } Row( modifier = Modifier .fillMaxWidth() .padding(top = 16.dp), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.SpaceBetween ) { Button( onClick = onToggleRecording, modifier = Modifier.weight(1f), content = { Text(if (isRecording) "Stop Transcription" else "Start Transcription") } ) Spacer(modifier = Modifier.width(16.dp)) Text( text = connectionStatus, style = MaterialTheme.typography.bodyMedium, fontWeight = FontWeight.Bold, color = when (connectionStatus) { "Server Ready" -> Color(0xFF4CAF50) // Green "Error" -> MaterialTheme.colorScheme.error "Connecting" -> Color(0xFFFFA000) // Amber else -> MaterialTheme.colorScheme.onSurfaceVariant } ) } } } } @Composable fun SettingsDialog( initialSettings: HandshakeSettings, onDismiss: () -> Unit, onSave: (HandshakeSettings) -> Unit ) { var host by remember { mutableStateOf(initialSettings.host) } var port by remember { mutableStateOf(initialSettings.port) } var task by remember { mutableStateOf(initialSettings.task) } var model by remember { mutableStateOf(initialSettings.model) } var language by remember { mutableStateOf(initialSettings.language) } var useVad by remember { mutableStateOf(initialSettings.useVad) } var audioFormat by remember { mutableStateOf(initialSettings.audioFormat) } var sendLastNSegments by remember { mutableStateOf(initialSettings.sendLastNSegments.toString()) } var noSpeechThresh by remember { mutableStateOf(initialSettings.noSpeechThresh.toString()) } var clipAudio by remember { mutableStateOf(initialSettings.clipAudio) } var sameOutputThreshold by remember { mutableStateOf(initialSettings.sameOutputThreshold.toString()) } var enableTranslation by remember { mutableStateOf(initialSettings.enableTranslation) } var targetLanguage by remember { mutableStateOf(initialSettings.targetLanguage) } var hotwords by remember { mutableStateOf(initialSettings.hotwords ?: "") } var enableDiarization by remember { mutableStateOf(initialSettings.enableDiarization) } var maxSpeakers by remember { mutableStateOf(initialSettings.maxSpeakers.toString()) } var wordTimestamps by remember { mutableStateOf(initialSettings.wordTimestamps) } AlertDialog( onDismissRequest = onDismiss, title = { Text("Server & Handshake Settings") }, text = { Column(modifier = Modifier.verticalScroll(rememberScrollState())) { Text("Network", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold) OutlinedTextField(value = host, onValueChange = { host = it }, label = { Text("Host / IP") }, modifier = Modifier.fillMaxWidth()) OutlinedTextField(value = port, onValueChange = { port = it }, label = { Text("Port") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) Text("Whisper Configuration", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold) OutlinedTextField(value = task, onValueChange = { task = it }, label = { Text("Task") }, modifier = Modifier.fillMaxWidth()) OutlinedTextField(value = model, onValueChange = { model = it }, label = { Text("Model") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) OutlinedTextField(value = language, onValueChange = { language = it }, label = { Text("Language") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) OutlinedTextField(value = audioFormat, onValueChange = { audioFormat = it }, label = { Text("Audio Format") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) OutlinedTextField(value = hotwords, onValueChange = { hotwords = it }, label = { Text("Hotwords (Optional)") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) { Text("Use VAD", modifier = Modifier.weight(1f)) Switch(checked = useVad, onCheckedChange = { useVad = it }) } Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) { Text("Clip Audio", modifier = Modifier.weight(1f)) Switch(checked = clipAudio, onCheckedChange = { clipAudio = it }) } Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(top = 8.dp)) { Text("Word Timestamps", modifier = Modifier.weight(1f)) Switch(checked = wordTimestamps, onCheckedChange = { wordTimestamps = it }) } HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) Text("Thresholds & Segments", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold) OutlinedTextField(value = sendLastNSegments, onValueChange = { sendLastNSegments = it }, label = { Text("Send Last N Segments") }, modifier = Modifier.fillMaxWidth()) OutlinedTextField(value = noSpeechThresh, onValueChange = { noSpeechThresh = it }, label = { Text("No Speech Threshold") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) OutlinedTextField(value = sameOutputThreshold, onValueChange = { sameOutputThreshold = it }, label = { Text("Same Output Threshold") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) Text("Translation", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold) Row(verticalAlignment = Alignment.CenterVertically) { Text("Enable Translation", modifier = Modifier.weight(1f)) Switch(checked = enableTranslation, onCheckedChange = { enableTranslation = it }) } OutlinedTextField(value = targetLanguage, onValueChange = { targetLanguage = it }, label = { Text("Target Language") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) Text("Diarization", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold) Row(verticalAlignment = Alignment.CenterVertically) { Text("Enable Diarization", modifier = Modifier.weight(1f)) Switch(checked = enableDiarization, onCheckedChange = { enableDiarization = it }) } OutlinedTextField(value = maxSpeakers, onValueChange = { maxSpeakers = it }, label = { Text("Max Speakers") }, modifier = Modifier.fillMaxWidth().padding(top = 8.dp)) } }, confirmButton = { TextButton(onClick = { onSave( HandshakeSettings( host = host, port = port, task = task, model = model, language = language, useVad = useVad, audioFormat = audioFormat, sendLastNSegments = sendLastNSegments.toIntOrNull() ?: SettingsManager.DEFAULT_SEND_LAST_N_SEGMENTS, noSpeechThresh = noSpeechThresh.toDoubleOrNull() ?: SettingsManager.DEFAULT_NO_SPEECH_THRESH, clipAudio = clipAudio, sameOutputThreshold = sameOutputThreshold.toIntOrNull() ?: SettingsManager.DEFAULT_SAME_OUTPUT_THRESHOLD, enableTranslation = enableTranslation, targetLanguage = targetLanguage, hotwords = hotwords.takeIf { it.isNotBlank() }, enableDiarization = enableDiarization, maxSpeakers = maxSpeakers.toIntOrNull() ?: SettingsManager.DEFAULT_MAX_SPEAKERS, wordTimestamps = wordTimestamps ) ) }) { Text("Save") } }, dismissButton = { TextButton(onClick = onDismiss) { Text("Cancel") } } ) } @Preview(showBackground = true) @Composable fun WhisperAppPreview() { WhisperClient2Theme { WhisperApp( transcriptText = "Transcription will appear here...", connectionStatus = "Disconnected" ) } }