Merge pull request #382 from ParkMazorika/main
Add iOS client for WhisperLive (Audio-Transcription-iOS)
This commit is contained in:
makaveli
@@ -0,0 +1,229 @@
|
||||
// AudioStream.swift
|
||||
// Lecture2Quiz
|
||||
//
|
||||
// Created by ParkMazorika on 4/27/25.
|
||||
//
|
||||
|
||||
import AVFoundation
|
||||
|
||||
/// Streams audio input to a WebSocket after converting and normalizing.
|
||||
class AudioStreamer {
|
||||
private let engine = AVAudioEngine()
|
||||
private let inputNode: AVAudioInputNode
|
||||
private var inputFormat: AVAudioFormat?
|
||||
private var isPaused: Bool = false
|
||||
private var audioWebSocket: AudioWebSocket?
|
||||
private var partialBuffer = Data()
|
||||
private var isStreaming: Bool = false
|
||||
|
||||
private var bufferSize: AVAudioFrameCount = 1600 // ~100ms of audio
|
||||
private var sampleRate: Double = 16000
|
||||
private var channels: UInt32 = 1
|
||||
|
||||
private var converter: AVAudioConverter?
|
||||
|
||||
init(webSocket: AudioWebSocket) {
|
||||
self.inputNode = engine.inputNode
|
||||
self.audioWebSocket = webSocket
|
||||
|
||||
let inputFormat = inputNode.outputFormat(forBus: 0)
|
||||
print("Input format: \(inputFormat)")
|
||||
|
||||
let outputFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatInt16,
|
||||
sampleRate: 16000,
|
||||
channels: 1,
|
||||
interleaved: true
|
||||
)!
|
||||
|
||||
self.converter = AVAudioConverter(from: inputFormat, to: outputFormat)
|
||||
self.inputFormat = outputFormat
|
||||
}
|
||||
|
||||
/// Configures the audio session for recording.
|
||||
func configureAudioSession() {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
do {
|
||||
try session.setCategory(.playAndRecord, mode: .default, options: [.allowBluetooth, .defaultToSpeaker])
|
||||
try session.setPreferredSampleRate(48000)
|
||||
try session.setPreferredInputNumberOfChannels(1)
|
||||
try session.setMode(.videoChat)
|
||||
try session.setActive(true, options: .notifyOthersOnDeactivation)
|
||||
sampleRate = session.sampleRate
|
||||
channels = UInt32(session.inputNumberOfChannels)
|
||||
print("Sample rate: \(sampleRate)")
|
||||
print("Input channels: \(channels)")
|
||||
} catch {
|
||||
print("Failed to configure audio session: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts capturing and streaming audio data.
|
||||
func startStreaming() {
|
||||
guard !isStreaming else {
|
||||
print("Already streaming.")
|
||||
return
|
||||
}
|
||||
|
||||
configureAudioSession()
|
||||
|
||||
let format = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: 48000,
|
||||
channels: channels,
|
||||
interleaved: true
|
||||
)
|
||||
|
||||
guard let hardwareFormat = format else {
|
||||
print("Failed to create audio format.")
|
||||
return
|
||||
}
|
||||
|
||||
self.inputFormat = hardwareFormat
|
||||
|
||||
inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: hardwareFormat) { [weak self] buffer, _ in
|
||||
self?.processAudioBuffer(buffer)
|
||||
}
|
||||
|
||||
do {
|
||||
try engine.start()
|
||||
isStreaming = true
|
||||
print("AVAudioEngine started.")
|
||||
} catch {
|
||||
print("Failed to start AVAudioEngine: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts and sends the audio buffer to the server via WebSocket.
|
||||
func processAudioBuffer(_ buffer: AVAudioPCMBuffer) {
|
||||
guard let converter = self.converter else {
|
||||
print("Audio converter is nil.")
|
||||
return
|
||||
}
|
||||
|
||||
if let floatChannelData = buffer.floatChannelData {
|
||||
let frameLength = Int(buffer.frameLength)
|
||||
let channelData = Array(UnsafeBufferPointer(start: floatChannelData.pointee, count: frameLength))
|
||||
let rms = sqrt(channelData.map { $0 * $0 }.reduce(0, +) / Float(frameLength))
|
||||
print("Audio RMS: \(rms)")
|
||||
if rms < 0.001 {
|
||||
print("Warning: Input volume is too low.")
|
||||
}
|
||||
}
|
||||
|
||||
let outputFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatInt16,
|
||||
sampleRate: 16000,
|
||||
channels: 1,
|
||||
interleaved: true
|
||||
)!
|
||||
|
||||
guard let newBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: 1600) else {
|
||||
print("Failed to allocate PCM buffer.")
|
||||
return
|
||||
}
|
||||
|
||||
let inputBlock: AVAudioConverterInputBlock = { _, outStatus in
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
|
||||
var error: NSError?
|
||||
converter.convert(to: newBuffer, error: &error, withInputFrom: inputBlock)
|
||||
|
||||
if let error = error {
|
||||
print("Audio conversion failed: \(error.localizedDescription)")
|
||||
return
|
||||
}
|
||||
|
||||
print("Converted buffer frameLength: \(newBuffer.frameLength), sampleRate: \(newBuffer.format.sampleRate)")
|
||||
|
||||
if let audioData = convertToFloat32BytesLikePython(newBuffer) {
|
||||
var completeData = partialBuffer + audioData
|
||||
let chunkSize = 4096
|
||||
|
||||
while completeData.count >= chunkSize {
|
||||
let chunk = completeData.prefix(chunkSize)
|
||||
audioWebSocket?.sendDataToServer(chunk)
|
||||
print("Sent 4096 bytes of audio.")
|
||||
completeData.removeFirst(chunkSize)
|
||||
}
|
||||
|
||||
partialBuffer = completeData
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the audio buffer to Float32 Data with RMS normalization and soft clipping.
|
||||
func convertToFloat32BytesLikePython(_ buffer: AVAudioPCMBuffer) -> Data? {
|
||||
guard let int16ChannelData = buffer.int16ChannelData else {
|
||||
print("int16ChannelData is nil.")
|
||||
return nil
|
||||
}
|
||||
|
||||
let frameLength = Int(buffer.frameLength)
|
||||
let channelPointer = int16ChannelData.pointee
|
||||
|
||||
var floatArray = [Float32](repeating: 0, count: frameLength)
|
||||
for i in 0..<frameLength {
|
||||
let int16Value = channelPointer[i]
|
||||
floatArray[i] = Float32(Int16(littleEndian: int16Value)) / 32768.0
|
||||
}
|
||||
|
||||
let rms = sqrt(floatArray.map { $0 * $0 }.reduce(0, +) / Float(frameLength))
|
||||
let targetRMS: Float32 = 0.25
|
||||
let gain = targetRMS / max(rms, 0.00001)
|
||||
|
||||
print("Original RMS: \(rms), applied gain: \(gain)")
|
||||
|
||||
for i in 0..<frameLength {
|
||||
let scaled = floatArray[i] * gain
|
||||
let clipped = tanh(scaled * 3.0)
|
||||
floatArray[i] = clipped
|
||||
}
|
||||
|
||||
let floatData = Data(bytes: floatArray, count: frameLength * MemoryLayout<Float32>.size)
|
||||
|
||||
if let minVal = floatArray.min(), let maxVal = floatArray.max() {
|
||||
print("Float32 value range after normalization: \(minVal)...\(maxVal)")
|
||||
}
|
||||
|
||||
print("Converted to Float32 data: \(floatData.count) bytes")
|
||||
return floatData
|
||||
}
|
||||
|
||||
/// Pauses audio streaming by removing the input tap.
|
||||
func pauseStreaming() {
|
||||
guard !isPaused else { return }
|
||||
inputNode.removeTap(onBus: 0)
|
||||
isPaused = true
|
||||
print("Audio streaming paused.")
|
||||
}
|
||||
|
||||
/// Resumes audio streaming by reinstalling the input tap.
|
||||
func resumeStreaming() {
|
||||
guard isPaused else { return }
|
||||
guard let inputFormat = inputFormat else {
|
||||
print("inputFormat is nil.")
|
||||
return
|
||||
}
|
||||
|
||||
inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: inputFormat) { [weak self] buffer, _ in
|
||||
self?.processAudioBuffer(buffer)
|
||||
}
|
||||
isPaused = false
|
||||
print("Audio streaming resumed.")
|
||||
}
|
||||
|
||||
/// Stops the AVAudioEngine and resets streaming state.
|
||||
func stopStreaming() {
|
||||
guard isStreaming else {
|
||||
print("Already stopped.")
|
||||
return
|
||||
}
|
||||
|
||||
inputNode.removeTap(onBus: 0)
|
||||
engine.stop()
|
||||
isStreaming = false
|
||||
print("AVAudioEngine stopped.")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
//
|
||||
// RecordingViewModel.swift
|
||||
// Lecture2Quiz
|
||||
//
|
||||
// Created by ParkMazorika on 4/27/25.
|
||||
//
|
||||
|
||||
|
||||
import Foundation
|
||||
|
||||
/// WebSocket client that connects to a transcription server and handles streaming, JSON messages, and retries.
|
||||
class AudioWebSocket: NSObject, URLSessionWebSocketDelegate {
|
||||
private var webSocketTask: URLSessionWebSocketTask?
|
||||
private var urlSession: URLSession!
|
||||
private let host: String
|
||||
private let port: Int
|
||||
private var retryCount = 0
|
||||
private let maxRetries = 3
|
||||
private var uid: String
|
||||
private let modelSize: String
|
||||
private var pingTimer: Timer?
|
||||
private var processedTexts = Set<String>()
|
||||
|
||||
var onServerReady: (() -> Void)?
|
||||
var onTranscriptionReceived: ((String) -> Void)?
|
||||
|
||||
init(host: String, port: Int, modelSize: String = "medium") {
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.uid = UUID().uuidString
|
||||
self.modelSize = modelSize
|
||||
super.init()
|
||||
|
||||
self.urlSession = URLSession(
|
||||
configuration: .default,
|
||||
delegate: self,
|
||||
delegateQueue: .main
|
||||
)
|
||||
connect()
|
||||
}
|
||||
|
||||
/// Establishes a WebSocket connection with the configured server.
|
||||
private func connect() {
|
||||
guard retryCount <= maxRetries else {
|
||||
print("Maximum reconnect attempts exceeded.")
|
||||
return
|
||||
}
|
||||
|
||||
let socketURL = port == 443 || port == 80
|
||||
? "wss://\(host)"
|
||||
: "wss://\(host):\(port)"
|
||||
|
||||
guard let url = URL(string: socketURL) else {
|
||||
print("Invalid URL: \(socketURL)")
|
||||
return
|
||||
}
|
||||
|
||||
webSocketTask = urlSession.webSocketTask(with: url)
|
||||
webSocketTask?.resume()
|
||||
print("Attempting WebSocket connection: \(socketURL)")
|
||||
|
||||
listen()
|
||||
sendInitialJSON()
|
||||
startPing()
|
||||
}
|
||||
|
||||
/// Sends the initial JSON payload to identify and configure the session.
|
||||
private func sendInitialJSON() {
|
||||
let jsonPayload: [String: Any] = [
|
||||
"uid": uid,
|
||||
"language": "en",
|
||||
"task": "transcribe",
|
||||
"model": modelSize,
|
||||
"use_vad": true,
|
||||
"max_clients": 4,
|
||||
"max_connection_time": 600
|
||||
]
|
||||
|
||||
do {
|
||||
let jsonData = try JSONSerialization.data(withJSONObject: jsonPayload, options: [])
|
||||
let jsonString = String(data: jsonData, encoding: .utf8) ?? ""
|
||||
print("Sending config JSON: \(jsonString)")
|
||||
|
||||
webSocketTask?.send(.string(jsonString)) { [weak self] error in
|
||||
if let error = error {
|
||||
print("Failed to send config JSON: \(error.localizedDescription)")
|
||||
self?.reconnect()
|
||||
} else {
|
||||
print("Config JSON sent successfully.")
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
print("JSON serialization error: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends audio data to the server.
|
||||
func sendDataToServer(_ data: Data) {
|
||||
guard isConnected else {
|
||||
print("Not connected - skipping data send.")
|
||||
reconnect()
|
||||
return
|
||||
}
|
||||
|
||||
webSocketTask?.send(.data(data)) { [weak self] error in
|
||||
if let error = error {
|
||||
print("Failed to send audio data: \(error.localizedDescription)")
|
||||
self?.reconnect()
|
||||
} else {
|
||||
print("Sent audio data: \(data.count) bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the WebSocket is currently connected.
|
||||
internal var isConnected: Bool {
|
||||
webSocketTask?.state == .running
|
||||
}
|
||||
|
||||
/// Attempts reconnection with exponential backoff.
|
||||
private func reconnect() {
|
||||
retryCount += 1
|
||||
stopPing()
|
||||
let delay = min(5.0, pow(2.0, Double(retryCount)))
|
||||
|
||||
DispatchQueue.global().asyncAfter(deadline: .now() + delay) { [weak self] in
|
||||
print("Reconnecting... (\(self?.retryCount ?? 0)/\(self?.maxRetries ?? 0))")
|
||||
self?.connect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts listening for incoming messages from the server.
|
||||
private func listen() {
|
||||
webSocketTask?.receive { [weak self] result in
|
||||
switch result {
|
||||
case .success(let message):
|
||||
self?.handleMessage(message)
|
||||
self?.listen()
|
||||
case .failure(let error):
|
||||
print("Receive error: \(error.localizedDescription)")
|
||||
self?.reconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles incoming WebSocket messages (text or binary).
|
||||
private func handleMessage(_ message: URLSessionWebSocketTask.Message) {
|
||||
switch message {
|
||||
case .data(let data):
|
||||
print("Received binary data: \(data.count) bytes")
|
||||
|
||||
case .string(let text):
|
||||
print("Received text message: \(text)")
|
||||
|
||||
guard let data = text.data(using: .utf8) else { return }
|
||||
|
||||
do {
|
||||
if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
||||
if let status = json["status"] as? String {
|
||||
handleStatusMessage(status: status, message: json["message"] as? String)
|
||||
return
|
||||
}
|
||||
|
||||
if let message = json["message"] as? String, message == "SERVER_READY" {
|
||||
print("Server is ready.")
|
||||
onServerReady?()
|
||||
return
|
||||
}
|
||||
|
||||
if let segments = json["segments"] as? [[String: Any]] {
|
||||
let wrapped = ["segments": segments]
|
||||
let segmentData = try JSONSerialization.data(withJSONObject: wrapped, options: [])
|
||||
let segmentString = String(data: segmentData, encoding: .utf8)!
|
||||
onTranscriptionReceived?(segmentString)
|
||||
print("Transcription segments forwarded.")
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
print("JSON parsing error: \(error.localizedDescription)")
|
||||
}
|
||||
|
||||
@unknown default:
|
||||
print("Unknown message type received.")
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles status message JSON from the server.
|
||||
private func handleStatusMessage(status: String, message: String?) {
|
||||
switch status {
|
||||
case "WAIT":
|
||||
print("Waiting: \(message ?? "")")
|
||||
case "ERROR":
|
||||
print("Error: \(message ?? "")")
|
||||
case "WARNING":
|
||||
print("Warning: \(message ?? "")")
|
||||
default:
|
||||
print("\(status): \(message ?? "")")
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends the "END_OF_AUDIO" signal to the server.
|
||||
func sendEndOfAudio() {
|
||||
guard isConnected else {
|
||||
print("Not connected - skipping END_OF_AUDIO.")
|
||||
return
|
||||
}
|
||||
|
||||
webSocketTask?.send(.string("END_OF_AUDIO")) { error in
|
||||
if let error = error {
|
||||
print("Failed to send END_OF_AUDIO: \(error.localizedDescription)")
|
||||
} else {
|
||||
print("END_OF_AUDIO sent.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gracefully closes the WebSocket connection.
|
||||
func closeConnection() {
|
||||
stopPing()
|
||||
webSocketTask?.cancel(with: .normalClosure, reason: nil)
|
||||
retryCount = maxRetries
|
||||
print("WebSocket closed.")
|
||||
}
|
||||
|
||||
/// Starts periodic ping to keep the WebSocket alive.
|
||||
private func startPing() {
|
||||
stopPing()
|
||||
pingTimer = Timer.scheduledTimer(withTimeInterval: 15.0, repeats: true) { [weak self] _ in
|
||||
self?.webSocketTask?.sendPing { error in
|
||||
if let error = error {
|
||||
print("Ping failed: \(error.localizedDescription)")
|
||||
} else {
|
||||
print("Ping sent successfully.")
|
||||
}
|
||||
}
|
||||
}
|
||||
RunLoop.main.add(pingTimer!, forMode: .common)
|
||||
}
|
||||
|
||||
/// Stops the periodic ping timer.
|
||||
private func stopPing() {
|
||||
pingTimer?.invalidate()
|
||||
pingTimer = nil
|
||||
}
|
||||
|
||||
/// Called when the WebSocket is closed by the server.
|
||||
func urlSession(_ session: URLSession,
|
||||
webSocketTask: URLSessionWebSocketTask,
|
||||
didCloseWith closeCode: URLSessionWebSocketTask.CloseCode,
|
||||
reason: Data?) {
|
||||
let reasonString = String(data: reason ?? Data(), encoding: .utf8) ?? "No reason"
|
||||
print("WebSocket closed - code: \(closeCode.rawValue), reason: \(reasonString)")
|
||||
stopPing()
|
||||
reconnect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//
|
||||
// ContentView.swift
|
||||
// WhisperLive_iOS_Client
|
||||
//
|
||||
// Created by ParkMazorika on 6/17/25.
|
||||
//
|
||||
|
||||
import SwiftUI
|
||||
|
||||
/// A standalone view for recording and real-time transcription display.
|
||||
struct RecordingView: View {
|
||||
var onDismiss: () -> Void
|
||||
@StateObject private var recordingViewModel = AudioViewModel()
|
||||
@State private var showSubmitView = false
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 0) {
|
||||
// Stop button (only visible when recording)
|
||||
HStack {
|
||||
Spacer()
|
||||
if recordingViewModel.isRecording {
|
||||
Button("Stop Recording") {
|
||||
recordingViewModel.stopRecording()
|
||||
recordingViewModel.finalizeTranscription()
|
||||
showSubmitView = true
|
||||
}
|
||||
.font(.headline)
|
||||
.padding()
|
||||
.foregroundColor(.gray)
|
||||
}
|
||||
}
|
||||
|
||||
// Transcription display
|
||||
ScrollView {
|
||||
VStack(spacing: 8) {
|
||||
ForEach(recordingViewModel.transcriptionList.indices, id: \.self) { index in
|
||||
Text(recordingViewModel.transcriptionList[index])
|
||||
.padding()
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(Color.gray.opacity(0.1))
|
||||
.cornerRadius(8)
|
||||
.font(.system(size: 14, weight: .semibold))
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
|
||||
Divider().padding(.top, 8)
|
||||
|
||||
// Timer and Record/Pause/Resume button
|
||||
VStack(spacing: 16) {
|
||||
Text(recordingViewModel.timeLabel)
|
||||
.font(.system(size: 40))
|
||||
|
||||
Button(action: {
|
||||
if recordingViewModel.isRecording {
|
||||
recordingViewModel.isPaused
|
||||
? recordingViewModel.resumeRecording()
|
||||
: recordingViewModel.pauseRecording()
|
||||
} else {
|
||||
recordingViewModel.startRecording()
|
||||
}
|
||||
}) {
|
||||
Image(systemName: recordingViewModel.isRecording
|
||||
? (recordingViewModel.isPaused ? "play.circle.fill" : "pause.circle.fill")
|
||||
: "mic.circle.fill")
|
||||
.font(.system(size: 50))
|
||||
.foregroundStyle(.black)
|
||||
}
|
||||
}
|
||||
.padding(.bottom, 40)
|
||||
}
|
||||
.padding(.top)
|
||||
.background(Color(.systemBackground))
|
||||
.overlay(
|
||||
Group {
|
||||
if recordingViewModel.isLoading {
|
||||
ZStack {
|
||||
Color.black.opacity(0.4).ignoresSafeArea()
|
||||
ProgressView("Processing...")
|
||||
.padding()
|
||||
.background(Color.white)
|
||||
.cornerRadius(10)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
.sheet(isPresented: $showSubmitView) {
|
||||
//anotherView
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#Preview("Recording View") {
|
||||
RecordingView {
|
||||
// Dummy dismiss handler
|
||||
print("RecordingView dismissed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Audio-Transcription-iOS
|
||||
|
||||
This is an iOS client for [WhisperLive](https://github.com/collabora/WhisperLive), a real-time speech-to-text server based on OpenAI Whisper.
|
||||
The app streams microphone audio to a WhisperLive server via WebSocket and displays live transcription results in real time.
|
||||
|
||||
> ⚠️ This client is designed to work specifically with the [WhisperLive Python WebSocket server](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server).
|
||||
> Make sure the server is running and reachable from your iOS device.
|
||||
|
||||
## Features
|
||||
|
||||
- Real-time microphone capture with AVAudioEngine
|
||||
- Streaming to WhisperLive backend using WebSocket
|
||||
- Displays transcription as segments arrive
|
||||
- Start / Pause / Resume / Stop recording with SwiftUI interface
|
||||
- Final transcription view on stop
|
||||
|
||||
## Requirements
|
||||
|
||||
- iOS 15.0+
|
||||
- Swift 5.8+
|
||||
- AVFoundation (for microphone)
|
||||
- Working WhisperLive WebSocket server
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Clone the repository (your fork):
|
||||
|
||||
```bash
|
||||
git clone https://github.com/yourusername/whisperlive.git
|
||||
cd whisperlive/Audio-Transcription-iOS
|
||||
```
|
||||
|
||||
2. Open the `.xcodeproj` or `.xcodeworkspace` in Xcode
|
||||
|
||||
3. Add the following to your `Info.plist`:
|
||||
|
||||
```xml
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>This app requires microphone access for transcription.</string>
|
||||
```
|
||||
|
||||
4. Run the app on a physical device (recommended)
|
||||
|
||||
## Running on a Physical Device (with Free Apple ID)
|
||||
|
||||
You can run this app on a real iPhone without a paid Apple Developer account. Follow these steps:
|
||||
|
||||
### 1. Register a Free Apple ID in Xcode
|
||||
|
||||
1. Open Xcode ▸ Settings… (or Preferences) ▸ **Accounts**
|
||||
2. Click the **+** button ▸ Select **Apple ID**
|
||||
3. Sign in with your Apple ID (a free one is fine)
|
||||
4. A "Personal Team" will be created automatically
|
||||
|
||||
> ✅ You can deploy up to 3 apps on a physical device using a free Apple ID with a 7-day provisioning profile.
|
||||
|
||||
---
|
||||
|
||||
### 2. Set Up Signing in Your Project
|
||||
|
||||
1. In Xcode, select your **project** in the Project Navigator
|
||||
2. Go to **TARGETS ▸ YourAppName ▸ Signing & Capabilities**
|
||||
3. Set **Team** to your Personal Team
|
||||
4. Set a unique **Bundle Identifier** (e.g., `com.yourname.whisperlive`)
|
||||
5. Make sure **Automatically manage signing** is checked
|
||||
6. If a red warning appears, click **"Resolve Issues"**
|
||||
|
||||
---
|
||||
|
||||
### 3. Connect and Trust Your iPhone
|
||||
|
||||
1. Connect your iPhone via USB
|
||||
2. When prompted, tap **“Trust This Computer”** on your iPhone
|
||||
3. Make sure your iPhone appears in Xcode's device list
|
||||
|
||||
---
|
||||
|
||||
### 4. Enable Developer Mode on iPhone
|
||||
|
||||
1. Press the **Build (▶︎)** button in Xcode
|
||||
2. Your iPhone will ask to enable **Developer Mode**
|
||||
3. On iPhone, go to:
|
||||
**Settings ▸ Privacy & Security ▸ Developer Mode**
|
||||
4. Enable it and restart the device if required
|
||||
|
||||
---
|
||||
|
||||
Now you can run and debug the app on your real device!
|
||||
|
||||
|
||||
|
||||
## Folder Structure
|
||||
```
|
||||
Audio-Transcription-iOS/
|
||||
├── AudioViewModel.swift
|
||||
├── AudioStreamer.swift
|
||||
├── AudioWebSocket.swift
|
||||
├── RecordingView.swift
|
||||
├── WhisperLive_iOS_ClientApp.swift
|
||||
├── Info.plist
|
||||
├── README.md
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
This iOS client is provided as an open-source example to complement WhisperLive's real-time transcription ecosystem.
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
//
|
||||
// RecordingViewModel.swift
|
||||
// Lecture2Quiz
|
||||
//
|
||||
// Created by ParkMazorika on 4/27/25.
|
||||
//
|
||||
|
||||
import AVFoundation
|
||||
import Combine
|
||||
|
||||
/// Represents a segment of transcribed audio with start/end timestamps and completion flag.
|
||||
struct TranscriptionSegment: Identifiable, Equatable {
|
||||
var id = UUID()
|
||||
var start: Double
|
||||
var end: Double
|
||||
var text: String
|
||||
var completed: Bool
|
||||
}
|
||||
|
||||
/// ViewModel responsible for managing audio recording and transcription logic.
|
||||
class AudioViewModel: ObservableObject {
|
||||
@Published var isRecording = false // Indicates if recording is active
|
||||
@Published var isPaused = false // Indicates if recording is currently paused
|
||||
@Published var timeLabel = "00:00" // Timer label formatted as mm:ss
|
||||
@Published var transcriptionList: [String] = [] // Live transcription output
|
||||
@Published var isLoading = false // True while waiting for server response
|
||||
@Published var finalScript: String = "" // Final script from completed segments
|
||||
|
||||
private var timer: Timer?
|
||||
private var elapsedTime: Int = 0
|
||||
|
||||
private var audioStreamer: AudioStreamer? // Handles audio capture and streaming
|
||||
private var audioWebSocket: AudioWebSocket? // Manages WebSocket communication
|
||||
|
||||
private var segments: [TranscriptionSegment] = [] // Stores all transcription segments
|
||||
|
||||
init() {}
|
||||
|
||||
/// Starts audio recording and initializes WebSocket + AVAudioEngine.
|
||||
func startRecording() {
|
||||
let audioAPIUrl = "your server url"
|
||||
audioWebSocket = AudioWebSocket(host: audioAPIUrl, port: 443)
|
||||
audioStreamer = AudioStreamer(webSocket: audioWebSocket!)
|
||||
|
||||
isLoading = true
|
||||
|
||||
// Handle server transcription message
|
||||
audioWebSocket?.onTranscriptionReceived = { [weak self] text in
|
||||
self?.handleRawTranscriptionJSON(text)
|
||||
}
|
||||
|
||||
// When server sends SERVER_READY
|
||||
audioWebSocket?.onServerReady = { [weak self] in
|
||||
guard let self = self else { return }
|
||||
DispatchQueue.main.async {
|
||||
self.isLoading = false
|
||||
self.isRecording = true
|
||||
self.isPaused = false
|
||||
self.timeLabel = "00:00"
|
||||
self.elapsedTime = 0
|
||||
self.startTimer()
|
||||
self.audioStreamer?.startStreaming()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pauses the recording and stops the timer.
|
||||
func pauseRecording() {
|
||||
isPaused = true
|
||||
audioStreamer?.pauseStreaming()
|
||||
timer?.invalidate()
|
||||
}
|
||||
|
||||
/// Resumes recording and restarts the timer.
|
||||
func resumeRecording() {
|
||||
isPaused = false
|
||||
audioStreamer?.resumeStreaming()
|
||||
startTimer()
|
||||
}
|
||||
|
||||
/// Stops recording and finalizes connection to server.
|
||||
func stopRecording() {
|
||||
isRecording = false
|
||||
isPaused = false
|
||||
timer?.invalidate()
|
||||
|
||||
audioStreamer?.stopStreaming()
|
||||
audioWebSocket?.sendEndOfAudio()
|
||||
audioWebSocket?.onTranscriptionReceived = nil
|
||||
audioWebSocket?.closeConnection()
|
||||
}
|
||||
|
||||
/// Starts the recording timer (1-second interval).
|
||||
private func startTimer() {
|
||||
timer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { _ in
|
||||
self.elapsedTime += 1
|
||||
let minutes = self.elapsedTime / 60
|
||||
let seconds = self.elapsedTime % 60
|
||||
self.timeLabel = String(format: "%02d:%02d", minutes, seconds)
|
||||
}
|
||||
}
|
||||
|
||||
/// Finalizes the transcription by joining all completed segments into one string.
|
||||
func finalizeTranscription() {
|
||||
isLoading = false
|
||||
let completedText = segments
|
||||
.filter { $0.completed }
|
||||
.map { $0.text.trimmingCharacters(in: .whitespaces) }
|
||||
.joined(separator: " ")
|
||||
finalScript = completedText
|
||||
print("Final transcript:\n\(finalScript)")
|
||||
}
|
||||
|
||||
/// Handles incoming JSON from the server and updates UI state.
|
||||
/// Supports both full JSON and raw string cases.
|
||||
func handleRawTranscriptionJSON(_ jsonString: String) {
|
||||
let trimmed = jsonString.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let data = trimmed.data(using: .utf8) else { return }
|
||||
|
||||
if trimmed.hasPrefix("{") {
|
||||
// Parse JSON containing segment list
|
||||
do {
|
||||
if let dict = try JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let segmentDicts = dict["segments"] as? [[String: Any]] {
|
||||
|
||||
for item in segmentDicts {
|
||||
guard let startStr = item["start"] as? String,
|
||||
let endStr = item["end"] as? String,
|
||||
let text = item["text"] as? String,
|
||||
let completed = item["completed"] as? Bool,
|
||||
let start = Double(startStr),
|
||||
let end = Double(endStr) else { continue }
|
||||
|
||||
let newSegment = TranscriptionSegment(start: start, end: end, text: text, completed: completed)
|
||||
|
||||
// Overwrite if already exists, else append
|
||||
if let index = self.segments.firstIndex(where: { $0.start == start }) {
|
||||
self.segments[index] = newSegment
|
||||
} else {
|
||||
self.segments.append(newSegment)
|
||||
}
|
||||
}
|
||||
|
||||
// Update the UI
|
||||
DispatchQueue.main.async {
|
||||
let completedTexts = self.segments
|
||||
.filter { $0.completed }
|
||||
.sorted(by: { $0.start < $1.start })
|
||||
.map { $0.text.trimmingCharacters(in: .whitespaces) }
|
||||
|
||||
let pendingText = self.segments
|
||||
.filter { !$0.completed }
|
||||
.sorted(by: { $0.start < $1.start })
|
||||
.map { $0.text.trimmingCharacters(in: .whitespaces) }
|
||||
.last ?? ""
|
||||
|
||||
self.transcriptionList = completedTexts + (pendingText.isEmpty ? [] : [pendingText])
|
||||
self.finalScript = self.transcriptionList.joined(separator: " ")
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
print("JSON parsing error: \(error)")
|
||||
}
|
||||
} else {
|
||||
// Handle raw text line
|
||||
DispatchQueue.main.async {
|
||||
if self.transcriptionList.last != trimmed {
|
||||
self.transcriptionList.append(trimmed)
|
||||
self.finalScript = self.transcriptionList.joined(separator: " ")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>This app requires microphone access for voice transcription.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,20 @@
|
||||
//
|
||||
// WhisperLive_iOS_ClientApp.swift
|
||||
// WhisperLive_iOS_Client
|
||||
//
|
||||
// Created by 바견규 on 6/17/25.
|
||||
//
|
||||
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct WhisperLive_iOS_ClientApp: App {
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
RecordingView {
|
||||
// Handle dismiss action here, or leave it empty for now
|
||||
print("RecordingView dismissed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -147,6 +147,12 @@ client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/b
|
||||
- Run the server with your desired backend as shown [here](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server).
|
||||
- Transcribe audio directly from your browser using our Chrome or Firefox extensions. Refer to [Audio-Transcription-Chrome](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Chrome#readme) and https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md
|
||||
|
||||
## iOS Client
|
||||
|
||||
Use WhisperLive on iOS with our native iOS client.
|
||||
Refer to [`ios-client`](https://github.com/collabora/WhisperLive/tree/main/Audio-Transcription-iOS) and [`ios-client/README.md`](https://github.com/collabora/WhisperLive/blob/main/Audio-Transcription-iOS/README.md) for setup and usage instructions.
|
||||
|
||||
|
||||
## Whisper Live Server in Docker
|
||||
- GPU
|
||||
- Faster-Whisper
|
||||
|
||||
Reference in New Issue
Block a user