First commit of Whisper client
This commit is contained in:
@@ -0,0 +1,411 @@
|
||||
package com.objectbrokers.whisperclient2
|
||||
|
||||
import android.Manifest
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.pm.PackageManager
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioRecord
|
||||
import android.media.MediaRecorder
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import android.widget.Toast
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Scaffold
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
|
||||
//import androidx.privacysandbox.tools.core.generator.build
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import okhttp3.WebSocket
|
||||
import okio.ByteString
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocketListener
|
||||
import com.objectbrokers.whisperclient2.ui.theme.WhisperClient2Theme
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.withContext
|
||||
import okio.ByteString.Companion.toByteString
|
||||
import org.json.JSONObject
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.UUID
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
const val TAG = "SpeechClient"
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
|
||||
private var recordingJob: Job? = null
|
||||
private var audioCaptureJob: Job? = null
|
||||
// private var isActive: Boolean = false
|
||||
private lateinit var audioRecord: AudioRecord
|
||||
private var webSocket: WebSocket? = null
|
||||
private var clientUid: UUID? = null
|
||||
private var bufferSize: Int = 0
|
||||
private var buffer: ByteArray? = null
|
||||
private var isRecording by mutableStateOf(false)
|
||||
private var transcriptText by mutableStateOf("Transcription will appear here...")
|
||||
private val client = OkHttpClient()
|
||||
private val requestPermissionLauncher = registerForActivityResult(
|
||||
ActivityResultContracts.RequestPermission()
|
||||
) { isGranted: Boolean ->
|
||||
if (isGranted) {
|
||||
// Permission granted! You can now call startRecording()
|
||||
} else {
|
||||
// Permission denied. Show a message to the user.
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
enableEdgeToEdge()
|
||||
setContent {
|
||||
WhisperClient2Theme {
|
||||
WhisperApp(transcriptText = transcriptText,
|
||||
isRecording = isRecording) {
|
||||
// this block IS the onToggleRecording lambda
|
||||
if (isRecording)
|
||||
{
|
||||
stopRecording()
|
||||
isRecording = false
|
||||
}
|
||||
else
|
||||
{
|
||||
startRecording(); isRecording = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
when {
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED -> {
|
||||
// Already have permission
|
||||
}
|
||||
else -> {
|
||||
// Ask for it
|
||||
requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class AudioInitializationException(message: String) : Exception(message)
|
||||
|
||||
@SuppressLint("MissingPermission") // Ensure runtime check is completed beforehand
|
||||
fun createAudioRecorder(): AudioRecord {
|
||||
// 1. Get the minimum buffer size required for the hardware configuration
|
||||
val minBufferSize = AudioRecord.getMinBufferSize(
|
||||
16000,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
)
|
||||
|
||||
// 2. Build and return the AudioRecord instance
|
||||
var recorder: AudioRecord? = null;
|
||||
recorder = AudioRecord.Builder()
|
||||
.setAudioSource(MediaRecorder.AudioSource.MIC)
|
||||
.setAudioFormat(
|
||||
AudioFormat.Builder()
|
||||
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
||||
.setSampleRate(16000)
|
||||
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
|
||||
.build()
|
||||
)
|
||||
// Double or triple the min buffer size to prevent buffer overflow (over-runs)
|
||||
.setBufferSizeInBytes(minBufferSize * 2)
|
||||
.build()
|
||||
|
||||
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
|
||||
recorder.release()
|
||||
throw AudioInitializationException("AudioRecord failed to initialize — mic unavailable or permission denied")
|
||||
}
|
||||
return recorder
|
||||
}
|
||||
|
||||
fun startRecording() {
|
||||
recordingJob = lifecycleScope.launch(Dispatchers.IO) {
|
||||
var audioRecordOK = true
|
||||
bufferSize = AudioRecord.getMinBufferSize(
|
||||
16000,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
) * 2
|
||||
|
||||
buffer = ByteArray(bufferSize)
|
||||
// Create the AudioRecord instance
|
||||
try {
|
||||
audioRecord = createAudioRecorder()
|
||||
} catch (e: AudioInitializationException) {
|
||||
Log.e(TAG, "Audio init failed: ${e.message}")
|
||||
withContext(Dispatchers.Main) {
|
||||
Toast.makeText(this@MainActivity, "Audio init failed: ${e.message}", Toast.LENGTH_LONG).show()
|
||||
audioRecordOK = false
|
||||
}
|
||||
}
|
||||
if (audioRecordOK) {
|
||||
// Create the WebSocket instance
|
||||
val request = Request.Builder()
|
||||
.url("ws://100.81.165.11:9097")
|
||||
.build()
|
||||
val listener = object : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
// get a new client UUID (unique session ID) for each connection
|
||||
clientUid = UUID.randomUUID()
|
||||
// Build the handshake JSON using Android's built-in JSONObject.
|
||||
// Every field the server expects is here; safe to hardcode the ones
|
||||
// you don't need to expose in the UI.
|
||||
val handshake = JSONObject().apply {
|
||||
put("uid", clientUid.toString())
|
||||
put("task", "transcribe")
|
||||
put("model", "OpenVINO/whisper-medium-fp16-ov")
|
||||
put("language", "en")
|
||||
put("use_vad", false)
|
||||
put("audio_format", "int16")
|
||||
put("send_last_n_segments", 10)
|
||||
put("no_speech_thresh", 0.45)
|
||||
put("clip_audio", false)
|
||||
put("same_output_threshold", 10)
|
||||
put("enable_translation", false)
|
||||
put(
|
||||
"target_language",
|
||||
"fr"
|
||||
) // server requires this key even if translation is off
|
||||
put("hotwords", JSONObject.NULL)
|
||||
put("enable_diarization", false)
|
||||
put("max_speakers", 10)
|
||||
put("word_timestamps", false)
|
||||
}
|
||||
// send() with a String argument sends a WebSocket text frame
|
||||
webSocket.send(handshake.toString())
|
||||
}
|
||||
|
||||
// Handle onClosing, onFailure, etc.
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, text: String) {
|
||||
Log.i(TAG, "Received message: $text")
|
||||
val message = JSONObject(text)
|
||||
|
||||
// 1. Validate this message is meant for us.
|
||||
// Store the uid you sent in the handshake as a class member.
|
||||
if (message.optString("uid") != clientUid.toString()) {
|
||||
Log.e(TAG, "Received message with wrong uid — ignoring")
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Status messages — server full, error, or warning.
|
||||
// These arrive at any time, not just after the handshake.
|
||||
if (message.has("status")) {
|
||||
when (message.getString("status")) {
|
||||
"WAIT" -> {
|
||||
val minutes = message.getDouble("message").roundToInt()
|
||||
Log.i(TAG, "Server full. Estimated wait: $minutes minutes")
|
||||
// Optionally show this in the UI
|
||||
}
|
||||
|
||||
"ERROR" -> {
|
||||
Log.e(TAG, "Server error: ${message.getString("message")}")
|
||||
// Stop recording, show error in UI
|
||||
}
|
||||
|
||||
"WARNING" -> {
|
||||
Log.w(TAG, "Server warning: ${message.getString("message")}")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Control messages — check the "message" key for named signals.
|
||||
if (message.has("message")) {
|
||||
when (message.getString("message")) {
|
||||
|
||||
"SERVER_READY" -> {
|
||||
// This is the handshake acknowledgement you asked about.
|
||||
// Only start sending audio after receiving this.
|
||||
val backend =
|
||||
message.getString("backend") // e.g. "faster_whisper"
|
||||
Log.i(TAG, "Server ready, backend: $backend")
|
||||
runOnUiThread {
|
||||
// statusTextView.text = "Connected"
|
||||
}
|
||||
startAudioCapture() // <-- safe to start sending audio now
|
||||
}
|
||||
|
||||
"DISCONNECT" -> {
|
||||
// Server hung up because the session exceeded MAX_CONNECTION_TIME
|
||||
Log.i(TAG, "Server disconnected: session time limit reached")
|
||||
stopRecording()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Language detection — server tells us what language it detected.
|
||||
// Only relevant if you sent language=null in the handshake.
|
||||
if (message.has("language")) {
|
||||
val lang = message.getString("language")
|
||||
val prob = message.getDouble("language_prob")
|
||||
Log.i(TAG, "Detected language: $lang (probability: $prob)")
|
||||
return
|
||||
}
|
||||
|
||||
// 5. Transcript segments — the normal steady-state message during recording.
|
||||
if (message.has("segments")) {
|
||||
val segments = message.getJSONArray("segments")
|
||||
val sb = StringBuilder()
|
||||
for (i in 0 until segments.length()) {
|
||||
sb.append(segments.getJSONObject(i).getString("text").trim())
|
||||
if (i < segments.length() - 1) sb.append(" ")
|
||||
}
|
||||
Log.i(TAG, "Transcript: $sb")
|
||||
runOnUiThread {
|
||||
val text12: String = sb.toString()
|
||||
this@MainActivity.transcriptText = text12
|
||||
// transcriptTextView.text = transcriptText
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFailure(
|
||||
webSocket: WebSocket,
|
||||
t: Throwable,
|
||||
response: Response?
|
||||
) {
|
||||
Log.e(TAG, "WebSocket connection failed: ${t.message}", t)
|
||||
}
|
||||
}
|
||||
|
||||
webSocket = client.newWebSocket(request, listener)
|
||||
Log.i(TAG, "WebSocket: $webSocket")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun stopRecording() {
|
||||
stopAudioCapture()
|
||||
recordingJob?.cancel() // the ?. is Kotlin null-safety — no NPE if job is null
|
||||
webSocket?.close(1000, "Done")
|
||||
webSocket = null
|
||||
}
|
||||
|
||||
fun stopAudioCapture() {
|
||||
audioCaptureJob?.cancel()
|
||||
audioRecord.stop()
|
||||
audioRecord.release()
|
||||
}
|
||||
|
||||
fun startAudioCapture() {
|
||||
audioCaptureJob = lifecycleScope.launch(Dispatchers.IO) {
|
||||
audioRecord.startRecording();
|
||||
Log.i(TAG, "audioRecord.state: ${audioRecord.state}")
|
||||
while (isActive) {
|
||||
buffer?.let {
|
||||
val bytesRead = audioRecord.read(it, 0, bufferSize)
|
||||
Log.i(TAG, "bytesRead: $bytesRead")
|
||||
if (bytesRead > 0) {
|
||||
// Check RMS energy — if this is always near zero, AudioRecord
|
||||
// is returning silence regardless of what the mic hears
|
||||
val shorts = ShortArray(bytesRead / 2)
|
||||
ByteBuffer.wrap(it, 0, bytesRead)
|
||||
.order(ByteOrder.LITTLE_ENDIAN)
|
||||
.asShortBuffer()
|
||||
.get(shorts)
|
||||
val rms = Math.sqrt(shorts.map { s -> s.toDouble() * s }.average())
|
||||
Log.d(TAG, "Audio chunk RMS: $rms")
|
||||
webSocket?.send(it.toByteString(0, bytesRead))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun sendAudioPacket(webSocket: WebSocket, audioBytes: ByteArray) {
|
||||
// OkHttp's send() is overloaded:
|
||||
// send(String) → text frame
|
||||
// send(ByteString) → binary frame ← this is what we want
|
||||
// ByteString.of() wraps a ByteArray without copying if possible.
|
||||
webSocket.send(ByteString.of(*audioBytes))
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@Composable
|
||||
fun WhisperApp(text: String)
|
||||
{ // ... inside your scrollable column ... Text( text = text,
|
||||
// It is now tied to the 'transcriptText' variable style = MaterialTheme.typography.bodyLarge ) }
|
||||
*/
|
||||
@Composable
|
||||
fun WhisperApp(
|
||||
transcriptText: String,
|
||||
isRecording: Boolean = false,
|
||||
onToggleRecording: () -> Unit = {}
|
||||
) {
|
||||
Scaffold(
|
||||
modifier = Modifier.fillMaxSize()
|
||||
) { innerPadding ->
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.padding(innerPadding)
|
||||
.fillMaxSize()
|
||||
.padding(16.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally
|
||||
) {
|
||||
Text(
|
||||
text = transcriptText,
|
||||
style = MaterialTheme.typography.headlineMedium,
|
||||
fontWeight = FontWeight.Bold,
|
||||
modifier = Modifier.padding(bottom = 16.dp)
|
||||
)
|
||||
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.weight(1f)
|
||||
.fillMaxWidth()
|
||||
.verticalScroll(rememberScrollState())
|
||||
) {
|
||||
Text(
|
||||
text = "Transcription will appear here...",
|
||||
style = MaterialTheme.typography.bodyLarge
|
||||
)
|
||||
}
|
||||
|
||||
Button(
|
||||
onClick = onToggleRecording,
|
||||
modifier = Modifier.fillMaxWidth().padding(top = 16.dp),
|
||||
content = { Text(if (isRecording) "Stop Transcription" else "Start Transcription") }
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun WhisperAppPreview() {
|
||||
WhisperClient2Theme {
|
||||
WhisperApp("Transcription will appear here...")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user