From 179b56a2601f661e8a77c0af3638c074e5e27dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=94=EA=B2=AC=EA=B7=9C?= Date: Tue, 17 Jun 2025 01:19:56 +0900 Subject: [PATCH 1/4] Add iOS client for WhisperLive (Audio-Transcription-iOS) --- .DS_Store | Bin 0 -> 8196 bytes Audio-Transcription-iOS/AudioStream.swift | 229 ++++++++++++++++ Audio-Transcription-iOS/AudioWebSocket.swift | 256 ++++++++++++++++++ Audio-Transcription-iOS/ContentView.swift | 99 +++++++ Audio-Transcription-iOS/README.md | 60 ++++ .../RecordingViewModel.swift | 174 ++++++++++++ .../WhisperLive-iOS-Client-Info.plist | 8 + .../WhisperLive_iOS_ClientApp.swift | 20 ++ 8 files changed, 846 insertions(+) create mode 100644 .DS_Store create mode 100644 Audio-Transcription-iOS/AudioStream.swift create mode 100644 Audio-Transcription-iOS/AudioWebSocket.swift create mode 100644 Audio-Transcription-iOS/ContentView.swift create mode 100644 Audio-Transcription-iOS/README.md create mode 100644 Audio-Transcription-iOS/RecordingViewModel.swift create mode 100644 Audio-Transcription-iOS/WhisperLive-iOS-Client-Info.plist create mode 100644 Audio-Transcription-iOS/WhisperLive_iOS_ClientApp.swift diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..f737795c9a96f4f4f15e198525bafae43d2160dc GIT binary patch literal 8196 zcmeHM-A)rh6h2c5-3Blu{y;P)n|NE0qM*i`Qow}7pM)iPp{BNUl?}^o-EFC+A?dC9 z4!(ky`Xc%+Ug8e!ZR(6$wp0n318z%pPNunhbS4B*V>$r*C)%T`-j1}p>r zB?EkaaFMyJDA|`%Y#k^>2mqN!wMckI93U{Rk`*QUQVJ++%IJY9C{rN@lW?@VTphBa zWM4`NCnn*f>UtDo#)xAseJ@6L^zCHMB?(*AzYxp?jR-HqB#xEsaqF+(J#HdLCF-w#X9z;AN6 z=Os}XN3Xb9{&N1C2ROwysy8`?L<{Tcd~M(OTAO?yI3)qzz~; z(<(iHoP)DAvk0^iebhF93E_aRu?a|5gnJ+LdWDngxJlqT(5#7#CZRn(XY|;jx7i+r zBkRaQW_vN_FP4$6?Wm2c1ItHQ=e+9r=dp?L8Kig<3E0@g2)o$20@SqB7zbDptm{@r zvA}&G29~*CT-Tg1tk;B#fQ>8*oOgCE!+UmAhq!;kQHIiDxD4K&O{c`bh_o}qAw@unhc-46yvFw_3%>4t@%ngXFQci~ItaC$86*Qb16Ma2!&E`RFnq(5;Hpr6O~{69&kacV!5?fEaf;p~c^@f1DM literal 0 HcmV?d00001 diff --git a/Audio-Transcription-iOS/AudioStream.swift b/Audio-Transcription-iOS/AudioStream.swift new file mode 100644 index 0000000..6cfad52 --- /dev/null +++ b/Audio-Transcription-iOS/AudioStream.swift @@ -0,0 +1,229 @@ +// AudioStream.swift +// Lecture2Quiz +// +// Created by ParkMazorika on 4/27/25. +// + +import AVFoundation + +/// Streams audio input to a WebSocket after converting and normalizing. +class AudioStreamer { + private let engine = AVAudioEngine() + private let inputNode: AVAudioInputNode + private var inputFormat: AVAudioFormat? + private var isPaused: Bool = false + private var audioWebSocket: AudioWebSocket? + private var partialBuffer = Data() + private var isStreaming: Bool = false + + private var bufferSize: AVAudioFrameCount = 1600 // ~100ms of audio + private var sampleRate: Double = 16000 + private var channels: UInt32 = 1 + + private var converter: AVAudioConverter? + + init(webSocket: AudioWebSocket) { + self.inputNode = engine.inputNode + self.audioWebSocket = webSocket + + let inputFormat = inputNode.outputFormat(forBus: 0) + print("Input format: \(inputFormat)") + + let outputFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: 16000, + channels: 1, + interleaved: true + )! + + self.converter = AVAudioConverter(from: inputFormat, to: outputFormat) + self.inputFormat = outputFormat + } + + /// Configures the audio session for recording. + func configureAudioSession() { + let session = AVAudioSession.sharedInstance() + do { + try session.setCategory(.playAndRecord, mode: .default, options: [.allowBluetooth, .defaultToSpeaker]) + try session.setPreferredSampleRate(48000) + try session.setPreferredInputNumberOfChannels(1) + try session.setMode(.videoChat) + try session.setActive(true, options: .notifyOthersOnDeactivation) + sampleRate = session.sampleRate + channels = UInt32(session.inputNumberOfChannels) + print("Sample rate: \(sampleRate)") + print("Input channels: \(channels)") + } catch { + print("Failed to configure audio session: \(error.localizedDescription)") + } + } + + /// Starts capturing and streaming audio data. + func startStreaming() { + guard !isStreaming else { + print("Already streaming.") + return + } + + configureAudioSession() + + let format = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 48000, + channels: channels, + interleaved: true + ) + + guard let hardwareFormat = format else { + print("Failed to create audio format.") + return + } + + self.inputFormat = hardwareFormat + + inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: hardwareFormat) { [weak self] buffer, _ in + self?.processAudioBuffer(buffer) + } + + do { + try engine.start() + isStreaming = true + print("AVAudioEngine started.") + } catch { + print("Failed to start AVAudioEngine: \(error.localizedDescription)") + } + } + + /// Converts and sends the audio buffer to the server via WebSocket. + func processAudioBuffer(_ buffer: AVAudioPCMBuffer) { + guard let converter = self.converter else { + print("Audio converter is nil.") + return + } + + if let floatChannelData = buffer.floatChannelData { + let frameLength = Int(buffer.frameLength) + let channelData = Array(UnsafeBufferPointer(start: floatChannelData.pointee, count: frameLength)) + let rms = sqrt(channelData.map { $0 * $0 }.reduce(0, +) / Float(frameLength)) + print("Audio RMS: \(rms)") + if rms < 0.001 { + print("Warning: Input volume is too low.") + } + } + + let outputFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: 16000, + channels: 1, + interleaved: true + )! + + guard let newBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: 1600) else { + print("Failed to allocate PCM buffer.") + return + } + + let inputBlock: AVAudioConverterInputBlock = { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + + var error: NSError? + converter.convert(to: newBuffer, error: &error, withInputFrom: inputBlock) + + if let error = error { + print("Audio conversion failed: \(error.localizedDescription)") + return + } + + print("Converted buffer frameLength: \(newBuffer.frameLength), sampleRate: \(newBuffer.format.sampleRate)") + + if let audioData = convertToFloat32BytesLikePython(newBuffer) { + var completeData = partialBuffer + audioData + let chunkSize = 4096 + + while completeData.count >= chunkSize { + let chunk = completeData.prefix(chunkSize) + audioWebSocket?.sendDataToServer(chunk) + print("Sent 4096 bytes of audio.") + completeData.removeFirst(chunkSize) + } + + partialBuffer = completeData + } + } + + /// Converts the audio buffer to Float32 Data with RMS normalization and soft clipping. + func convertToFloat32BytesLikePython(_ buffer: AVAudioPCMBuffer) -> Data? { + guard let int16ChannelData = buffer.int16ChannelData else { + print("int16ChannelData is nil.") + return nil + } + + let frameLength = Int(buffer.frameLength) + let channelPointer = int16ChannelData.pointee + + var floatArray = [Float32](repeating: 0, count: frameLength) + for i in 0...size) + + if let minVal = floatArray.min(), let maxVal = floatArray.max() { + print("Float32 value range after normalization: \(minVal)...\(maxVal)") + } + + print("Converted to Float32 data: \(floatData.count) bytes") + return floatData + } + + /// Pauses audio streaming by removing the input tap. + func pauseStreaming() { + guard !isPaused else { return } + inputNode.removeTap(onBus: 0) + isPaused = true + print("Audio streaming paused.") + } + + /// Resumes audio streaming by reinstalling the input tap. + func resumeStreaming() { + guard isPaused else { return } + guard let inputFormat = inputFormat else { + print("inputFormat is nil.") + return + } + + inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: inputFormat) { [weak self] buffer, _ in + self?.processAudioBuffer(buffer) + } + isPaused = false + print("Audio streaming resumed.") + } + + /// Stops the AVAudioEngine and resets streaming state. + func stopStreaming() { + guard isStreaming else { + print("Already stopped.") + return + } + + inputNode.removeTap(onBus: 0) + engine.stop() + isStreaming = false + print("AVAudioEngine stopped.") + } +} diff --git a/Audio-Transcription-iOS/AudioWebSocket.swift b/Audio-Transcription-iOS/AudioWebSocket.swift new file mode 100644 index 0000000..caf2e38 --- /dev/null +++ b/Audio-Transcription-iOS/AudioWebSocket.swift @@ -0,0 +1,256 @@ +// +// RecordingViewModel.swift +// Lecture2Quiz +// +// Created by ParkMazorika on 4/27/25. +// + + +import Foundation + +/// WebSocket client that connects to a transcription server and handles streaming, JSON messages, and retries. +class AudioWebSocket: NSObject, URLSessionWebSocketDelegate { + private var webSocketTask: URLSessionWebSocketTask? + private var urlSession: URLSession! + private let host: String + private let port: Int + private var retryCount = 0 + private let maxRetries = 3 + private var uid: String + private let modelSize: String + private var pingTimer: Timer? + private var processedTexts = Set() + + var onServerReady: (() -> Void)? + var onTranscriptionReceived: ((String) -> Void)? + + init(host: String, port: Int, modelSize: String = "medium") { + self.host = host + self.port = port + self.uid = UUID().uuidString + self.modelSize = modelSize + super.init() + + self.urlSession = URLSession( + configuration: .default, + delegate: self, + delegateQueue: .main + ) + connect() + } + + /// Establishes a WebSocket connection with the configured server. + private func connect() { + guard retryCount <= maxRetries else { + print("Maximum reconnect attempts exceeded.") + return + } + + let socketURL = port == 443 || port == 80 + ? "wss://\(host)" + : "wss://\(host):\(port)" + + guard let url = URL(string: socketURL) else { + print("Invalid URL: \(socketURL)") + return + } + + webSocketTask = urlSession.webSocketTask(with: url) + webSocketTask?.resume() + print("Attempting WebSocket connection: \(socketURL)") + + listen() + sendInitialJSON() + startPing() + } + + /// Sends the initial JSON payload to identify and configure the session. + private func sendInitialJSON() { + let jsonPayload: [String: Any] = [ + "uid": uid, + "language": "en", + "task": "transcribe", + "model": modelSize, + "use_vad": true, + "max_clients": 4, + "max_connection_time": 600 + ] + + do { + let jsonData = try JSONSerialization.data(withJSONObject: jsonPayload, options: []) + let jsonString = String(data: jsonData, encoding: .utf8) ?? "" + print("Sending config JSON: \(jsonString)") + + webSocketTask?.send(.string(jsonString)) { [weak self] error in + if let error = error { + print("Failed to send config JSON: \(error.localizedDescription)") + self?.reconnect() + } else { + print("Config JSON sent successfully.") + } + } + } catch { + print("JSON serialization error: \(error.localizedDescription)") + } + } + + /// Sends audio data to the server. + func sendDataToServer(_ data: Data) { + guard isConnected else { + print("Not connected - skipping data send.") + reconnect() + return + } + + webSocketTask?.send(.data(data)) { [weak self] error in + if let error = error { + print("Failed to send audio data: \(error.localizedDescription)") + self?.reconnect() + } else { + print("Sent audio data: \(data.count) bytes") + } + } + } + + /// Returns true if the WebSocket is currently connected. + internal var isConnected: Bool { + webSocketTask?.state == .running + } + + /// Attempts reconnection with exponential backoff. + private func reconnect() { + retryCount += 1 + stopPing() + let delay = min(5.0, pow(2.0, Double(retryCount))) + + DispatchQueue.global().asyncAfter(deadline: .now() + delay) { [weak self] in + print("Reconnecting... (\(self?.retryCount ?? 0)/\(self?.maxRetries ?? 0))") + self?.connect() + } + } + + /// Starts listening for incoming messages from the server. + private func listen() { + webSocketTask?.receive { [weak self] result in + switch result { + case .success(let message): + self?.handleMessage(message) + self?.listen() + case .failure(let error): + print("Receive error: \(error.localizedDescription)") + self?.reconnect() + } + } + } + + /// Handles incoming WebSocket messages (text or binary). + private func handleMessage(_ message: URLSessionWebSocketTask.Message) { + switch message { + case .data(let data): + print("Received binary data: \(data.count) bytes") + + case .string(let text): + print("Received text message: \(text)") + + guard let data = text.data(using: .utf8) else { return } + + do { + if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + if let status = json["status"] as? String { + handleStatusMessage(status: status, message: json["message"] as? String) + return + } + + if let message = json["message"] as? String, message == "SERVER_READY" { + print("Server is ready.") + onServerReady?() + return + } + + if let segments = json["segments"] as? [[String: Any]] { + let wrapped = ["segments": segments] + let segmentData = try JSONSerialization.data(withJSONObject: wrapped, options: []) + let segmentString = String(data: segmentData, encoding: .utf8)! + onTranscriptionReceived?(segmentString) + print("Transcription segments forwarded.") + } + } + } catch { + print("JSON parsing error: \(error.localizedDescription)") + } + + @unknown default: + print("Unknown message type received.") + } + } + + /// Handles status message JSON from the server. + private func handleStatusMessage(status: String, message: String?) { + switch status { + case "WAIT": + print("Waiting: \(message ?? "")") + case "ERROR": + print("Error: \(message ?? "")") + case "WARNING": + print("Warning: \(message ?? "")") + default: + print("\(status): \(message ?? "")") + } + } + + /// Sends the "END_OF_AUDIO" signal to the server. + func sendEndOfAudio() { + guard isConnected else { + print("Not connected - skipping END_OF_AUDIO.") + return + } + + webSocketTask?.send(.string("END_OF_AUDIO")) { error in + if let error = error { + print("Failed to send END_OF_AUDIO: \(error.localizedDescription)") + } else { + print("END_OF_AUDIO sent.") + } + } + } + + /// Gracefully closes the WebSocket connection. + func closeConnection() { + stopPing() + webSocketTask?.cancel(with: .normalClosure, reason: nil) + retryCount = maxRetries + print("WebSocket closed.") + } + + /// Starts periodic ping to keep the WebSocket alive. + private func startPing() { + stopPing() + pingTimer = Timer.scheduledTimer(withTimeInterval: 15.0, repeats: true) { [weak self] _ in + self?.webSocketTask?.sendPing { error in + if let error = error { + print("Ping failed: \(error.localizedDescription)") + } else { + print("Ping sent successfully.") + } + } + } + RunLoop.main.add(pingTimer!, forMode: .common) + } + + /// Stops the periodic ping timer. + private func stopPing() { + pingTimer?.invalidate() + pingTimer = nil + } + + /// Called when the WebSocket is closed by the server. + func urlSession(_ session: URLSession, + webSocketTask: URLSessionWebSocketTask, + didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, + reason: Data?) { + let reasonString = String(data: reason ?? Data(), encoding: .utf8) ?? "No reason" + print("WebSocket closed - code: \(closeCode.rawValue), reason: \(reasonString)") + stopPing() + reconnect() + } +} diff --git a/Audio-Transcription-iOS/ContentView.swift b/Audio-Transcription-iOS/ContentView.swift new file mode 100644 index 0000000..787f2b9 --- /dev/null +++ b/Audio-Transcription-iOS/ContentView.swift @@ -0,0 +1,99 @@ +// +// ContentView.swift +// WhisperLive_iOS_Client +// +// Created by ParkMazorika on 6/17/25. +// + +import SwiftUI + +/// A standalone view for recording and real-time transcription display. +struct RecordingView: View { + var onDismiss: () -> Void + @StateObject private var recordingViewModel = AudioViewModel() + @State private var showSubmitView = false + + var body: some View { + VStack(spacing: 0) { + // Stop button (only visible when recording) + HStack { + Spacer() + if recordingViewModel.isRecording { + Button("Stop Recording") { + recordingViewModel.stopRecording() + recordingViewModel.finalizeTranscription() + showSubmitView = true + } + .font(.headline) + .padding() + .foregroundColor(.gray) + } + } + + // Transcription display + ScrollView { + VStack(spacing: 8) { + ForEach(recordingViewModel.transcriptionList.indices, id: \.self) { index in + Text(recordingViewModel.transcriptionList[index]) + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.gray.opacity(0.1)) + .cornerRadius(8) + .font(.system(size: 14, weight: .semibold)) + } + } + .padding(.horizontal) + } + + Divider().padding(.top, 8) + + // Timer and Record/Pause/Resume button + VStack(spacing: 16) { + Text(recordingViewModel.timeLabel) + .font(.system(size: 40)) + + Button(action: { + if recordingViewModel.isRecording { + recordingViewModel.isPaused + ? recordingViewModel.resumeRecording() + : recordingViewModel.pauseRecording() + } else { + recordingViewModel.startRecording() + } + }) { + Image(systemName: recordingViewModel.isRecording + ? (recordingViewModel.isPaused ? "play.circle.fill" : "pause.circle.fill") + : "mic.circle.fill") + .font(.system(size: 50)) + .foregroundStyle(.black) + } + } + .padding(.bottom, 40) + } + .padding(.top) + .background(Color(.systemBackground)) + .overlay( + Group { + if recordingViewModel.isLoading { + ZStack { + Color.black.opacity(0.4).ignoresSafeArea() + ProgressView("Processing...") + .padding() + .background(Color.white) + .cornerRadius(10) + } + } + } + ) + .sheet(isPresented: $showSubmitView) { + //anotherView + } + } +} + +#Preview("Recording View") { + RecordingView { + // Dummy dismiss handler + print("RecordingView dismissed") + } +} diff --git a/Audio-Transcription-iOS/README.md b/Audio-Transcription-iOS/README.md new file mode 100644 index 0000000..2a71b57 --- /dev/null +++ b/Audio-Transcription-iOS/README.md @@ -0,0 +1,60 @@ +# Audio-Transcription-iOS + +This is an iOS client for [WhisperLive](https://github.com/collabora/WhisperLive), a real-time speech-to-text server based on OpenAI Whisper. +The app streams microphone audio to a WhisperLive server via WebSocket and displays live transcription results in real time. + +> ⚠️ This client is designed to work specifically with the [WhisperLive Python WebSocket server](https://github.com/whisperlive/whisperlive). +> Make sure the server is running and reachable from your iOS device. + +## Features + +- Real-time microphone capture with AVAudioEngine +- Streaming to WhisperLive backend using WebSocket +- Displays transcription as segments arrive +- Start / Pause / Resume / Stop recording with SwiftUI interface +- Final transcription view on stop + +## Requirements + +- iOS 15.0+ +- Swift 5.8+ +- AVFoundation (for microphone) +- Working WhisperLive WebSocket server + +## Getting Started + +1. Clone the repository (your fork): + + ```bash + git clone https://github.com/yourusername/whisperlive.git + cd whisperlive/Audio-Transcription-iOS + ``` + +2. Open the `.xcodeproj` or `.xcodeworkspace` in Xcode + +3. Add the following to your `Info.plist`: + + ```xml + NSMicrophoneUsageDescription + This app requires microphone access for transcription. + ``` + +4. Run the app on a physical device (recommended) + +## Folder Structure +Audio-Transcription-iOS/ +├── AudioViewModel.swift +├── AudioStreamer.swift +├── AudioWebSocket.swift +├── RecordingView.swift +├── WhisperLive_iOS_ClientApp.swift +├── Info.plist +├── README.md + + +## License + +MIT +This iOS client is provided as an open-source example to complement WhisperLive's real-time transcription ecosystem. + + diff --git a/Audio-Transcription-iOS/RecordingViewModel.swift b/Audio-Transcription-iOS/RecordingViewModel.swift new file mode 100644 index 0000000..5192e43 --- /dev/null +++ b/Audio-Transcription-iOS/RecordingViewModel.swift @@ -0,0 +1,174 @@ +// +// RecordingViewModel.swift +// Lecture2Quiz +// +// Created by ParkMazorika on 4/27/25. +// + +import AVFoundation +import Combine + +/// Represents a segment of transcribed audio with start/end timestamps and completion flag. +struct TranscriptionSegment: Identifiable, Equatable { + var id = UUID() + var start: Double + var end: Double + var text: String + var completed: Bool +} + +/// ViewModel responsible for managing audio recording and transcription logic. +class AudioViewModel: ObservableObject { + @Published var isRecording = false // Indicates if recording is active + @Published var isPaused = false // Indicates if recording is currently paused + @Published var timeLabel = "00:00" // Timer label formatted as mm:ss + @Published var transcriptionList: [String] = [] // Live transcription output + @Published var isLoading = false // True while waiting for server response + @Published var finalScript: String = "" // Final script from completed segments + + private var timer: Timer? + private var elapsedTime: Int = 0 + + private var audioStreamer: AudioStreamer? // Handles audio capture and streaming + private var audioWebSocket: AudioWebSocket? // Manages WebSocket communication + + private var segments: [TranscriptionSegment] = [] // Stores all transcription segments + + init() {} + + /// Starts audio recording and initializes WebSocket + AVAudioEngine. + func startRecording() { + let audioAPIUrl = "your server url" + audioWebSocket = AudioWebSocket(host: audioAPIUrl, port: 443) + audioStreamer = AudioStreamer(webSocket: audioWebSocket!) + + isLoading = true + + // Handle server transcription message + audioWebSocket?.onTranscriptionReceived = { [weak self] text in + self?.handleRawTranscriptionJSON(text) + } + + // When server sends SERVER_READY + audioWebSocket?.onServerReady = { [weak self] in + guard let self = self else { return } + DispatchQueue.main.async { + self.isLoading = false + self.isRecording = true + self.isPaused = false + self.timeLabel = "00:00" + self.elapsedTime = 0 + self.startTimer() + self.audioStreamer?.startStreaming() + } + } + } + + /// Pauses the recording and stops the timer. + func pauseRecording() { + isPaused = true + audioStreamer?.pauseStreaming() + timer?.invalidate() + } + + /// Resumes recording and restarts the timer. + func resumeRecording() { + isPaused = false + audioStreamer?.resumeStreaming() + startTimer() + } + + /// Stops recording and finalizes connection to server. + func stopRecording() { + isRecording = false + isPaused = false + timer?.invalidate() + + audioStreamer?.stopStreaming() + audioWebSocket?.sendEndOfAudio() + audioWebSocket?.onTranscriptionReceived = nil + audioWebSocket?.closeConnection() + } + + /// Starts the recording timer (1-second interval). + private func startTimer() { + timer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { _ in + self.elapsedTime += 1 + let minutes = self.elapsedTime / 60 + let seconds = self.elapsedTime % 60 + self.timeLabel = String(format: "%02d:%02d", minutes, seconds) + } + } + + /// Finalizes the transcription by joining all completed segments into one string. + func finalizeTranscription() { + isLoading = false + let completedText = segments + .filter { $0.completed } + .map { $0.text.trimmingCharacters(in: .whitespaces) } + .joined(separator: " ") + finalScript = completedText + print("Final transcript:\n\(finalScript)") + } + + /// Handles incoming JSON from the server and updates UI state. + /// Supports both full JSON and raw string cases. + func handleRawTranscriptionJSON(_ jsonString: String) { + let trimmed = jsonString.trimmingCharacters(in: .whitespacesAndNewlines) + guard let data = trimmed.data(using: .utf8) else { return } + + if trimmed.hasPrefix("{") { + // Parse JSON containing segment list + do { + if let dict = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let segmentDicts = dict["segments"] as? [[String: Any]] { + + for item in segmentDicts { + guard let startStr = item["start"] as? String, + let endStr = item["end"] as? String, + let text = item["text"] as? String, + let completed = item["completed"] as? Bool, + let start = Double(startStr), + let end = Double(endStr) else { continue } + + let newSegment = TranscriptionSegment(start: start, end: end, text: text, completed: completed) + + // Overwrite if already exists, else append + if let index = self.segments.firstIndex(where: { $0.start == start }) { + self.segments[index] = newSegment + } else { + self.segments.append(newSegment) + } + } + + // Update the UI + DispatchQueue.main.async { + let completedTexts = self.segments + .filter { $0.completed } + .sorted(by: { $0.start < $1.start }) + .map { $0.text.trimmingCharacters(in: .whitespaces) } + + let pendingText = self.segments + .filter { !$0.completed } + .sorted(by: { $0.start < $1.start }) + .map { $0.text.trimmingCharacters(in: .whitespaces) } + .last ?? "" + + self.transcriptionList = completedTexts + (pendingText.isEmpty ? [] : [pendingText]) + self.finalScript = self.transcriptionList.joined(separator: " ") + } + } + } catch { + print("JSON parsing error: \(error)") + } + } else { + // Handle raw text line + DispatchQueue.main.async { + if self.transcriptionList.last != trimmed { + self.transcriptionList.append(trimmed) + self.finalScript = self.transcriptionList.joined(separator: " ") + } + } + } + } +} diff --git a/Audio-Transcription-iOS/WhisperLive-iOS-Client-Info.plist b/Audio-Transcription-iOS/WhisperLive-iOS-Client-Info.plist new file mode 100644 index 0000000..5902fc5 --- /dev/null +++ b/Audio-Transcription-iOS/WhisperLive-iOS-Client-Info.plist @@ -0,0 +1,8 @@ + + + + + NSMicrophoneUsageDescription + This app requires microphone access for voice transcription. + + diff --git a/Audio-Transcription-iOS/WhisperLive_iOS_ClientApp.swift b/Audio-Transcription-iOS/WhisperLive_iOS_ClientApp.swift new file mode 100644 index 0000000..2af33da --- /dev/null +++ b/Audio-Transcription-iOS/WhisperLive_iOS_ClientApp.swift @@ -0,0 +1,20 @@ +// +// WhisperLive_iOS_ClientApp.swift +// WhisperLive_iOS_Client +// +// Created by 바견규 on 6/17/25. +// + +import SwiftUI + +@main +struct WhisperLive_iOS_ClientApp: App { + var body: some Scene { + WindowGroup { + RecordingView { + // Handle dismiss action here, or leave it empty for now + print("RecordingView dismissed") + } + } + } +} From 2e5aae65853476a80944407585d5d27218dc9eec Mon Sep 17 00:00:00 2001 From: Park hyeon gyu <93672961+ParkMazorika@users.noreply.github.com> Date: Mon, 23 Jun 2025 20:50:39 +0900 Subject: [PATCH 2/4] Update Audio-Transcription-iOS/README.md Co-authored-by: makaveli <39617050+makaveli10@users.noreply.github.com> --- Audio-Transcription-iOS/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Audio-Transcription-iOS/README.md b/Audio-Transcription-iOS/README.md index 2a71b57..49bc17e 100644 --- a/Audio-Transcription-iOS/README.md +++ b/Audio-Transcription-iOS/README.md @@ -3,7 +3,7 @@ This is an iOS client for [WhisperLive](https://github.com/collabora/WhisperLive), a real-time speech-to-text server based on OpenAI Whisper. The app streams microphone audio to a WhisperLive server via WebSocket and displays live transcription results in real time. -> ⚠️ This client is designed to work specifically with the [WhisperLive Python WebSocket server](https://github.com/whisperlive/whisperlive). +> ⚠️ This client is designed to work specifically with the [WhisperLive Python WebSocket server](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server). > Make sure the server is running and reachable from your iOS device. ## Features From f3acfa2f18b157c76028998dd91a848c7ed0634a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=94=EA=B2=AC=EA=B7=9C?= Date: Sat, 28 Jun 2025 07:21:04 +0900 Subject: [PATCH 3/4] Add iOS client section to main README/ modify iOS README --- .DS_Store | Bin 8196 -> 8196 bytes Audio-Transcription-iOS/README.md | 51 +++++++++++++++++++++++++++++- README.md | 6 ++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.DS_Store b/.DS_Store index f737795c9a96f4f4f15e198525bafae43d2160dc..a63eb67cdcbd46fe107cfc6eabe3a27ef312fe18 100644 GIT binary patch delta 202 zcmZp1XmOa}&nUbxU^hRb@MazXaYj~I1_nmH$+rZoCMO9t3K^T~D41B9)aocyTN)Vv z*~XK<3KmW75SE$TDkLy@i?I0Q#lnJki(G4kjjw9P{L5`nUkNKl#`#tz`!8Dz`)oIr1k!T0g%POfTlXB zytn|Wdi&%&!9q3h>S|L%104l3LnOP*YHK+;M3wcegW|Jua`W;#CkKei3wL1}H#tH` zd-7@_fytAF#3$bn5}dqSNRIL9=37DqjBHF-f$HZ?4iGV(>@A`>d5s7+BdJz0c5N;Y VEoa=!F7b_J@*Uwks+n2B2mlSbYjgkr diff --git a/Audio-Transcription-iOS/README.md b/Audio-Transcription-iOS/README.md index 49bc17e..8e4455c 100644 --- a/Audio-Transcription-iOS/README.md +++ b/Audio-Transcription-iOS/README.md @@ -41,7 +41,56 @@ The app streams microphone audio to a WhisperLive server via WebSocket and displ 4. Run the app on a physical device (recommended) +## Running on a Physical Device (with Free Apple ID) + +You can run this app on a real iPhone without a paid Apple Developer account. Follow these steps: + +### 1. Register a Free Apple ID in Xcode + +1. Open Xcode ▸ Settings… (or Preferences) ▸ **Accounts** +2. Click the **+** button ▸ Select **Apple ID** +3. Sign in with your Apple ID (a free one is fine) +4. A "Personal Team" will be created automatically + +> ✅ You can deploy up to 3 apps on a physical device using a free Apple ID with a 7-day provisioning profile. + +--- + +### 2. Set Up Signing in Your Project + +1. In Xcode, select your **project** in the Project Navigator +2. Go to **TARGETS ▸ YourAppName ▸ Signing & Capabilities** +3. Set **Team** to your Personal Team +4. Set a unique **Bundle Identifier** (e.g., `com.yourname.whisperlive`) +5. Make sure **Automatically manage signing** is checked +6. If a red warning appears, click **"Resolve Issues"** + +--- + +### 3. Connect and Trust Your iPhone + +1. Connect your iPhone via USB +2. When prompted, tap **“Trust This Computer”** on your iPhone +3. Make sure your iPhone appears in Xcode's device list + +--- + +### 4. Enable Developer Mode on iPhone + +1. Press the **Build (▶︎)** button in Xcode +2. Your iPhone will ask to enable **Developer Mode** +3. On iPhone, go to: + **Settings ▸ Privacy & Security ▸ Developer Mode** +4. Enable it and restart the device if required + +--- + +Now you can run and debug the app on your real device! + + + ## Folder Structure +``` Audio-Transcription-iOS/ ├── AudioViewModel.swift ├── AudioStreamer.swift @@ -50,7 +99,7 @@ Audio-Transcription-iOS/ ├── WhisperLive_iOS_ClientApp.swift ├── Info.plist ├── README.md - +``` ## License diff --git a/README.md b/README.md index 306d8bc..49ed1bb 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,12 @@ client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/b - Run the server with your desired backend as shown [here](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server). - Transcribe audio directly from your browser using our Chrome or Firefox extensions. Refer to [Audio-Transcription-Chrome](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Chrome#readme) and https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md +## iOS Client + +Use WhisperLive on iOS with our native iOS client. +Refer to [`ios-client`](https://github.com/collabora/WhisperLive/tree/main/Audio-Transcription-iOS) and [`ios-client/README.md`](https://github.com/collabora/WhisperLive/blob/main/Audio-Transcription-iOS/README.md) for setup and usage instructions. + + ## Whisper Live Server in Docker - GPU - Faster-Whisper From d79e720b343914c0983f4acfd4ec5a04253de084 Mon Sep 17 00:00:00 2001 From: Park hyeon gyu <93672961+ParkMazorika@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:50:12 +0900 Subject: [PATCH 4/4] Delete .DS_Store --- .DS_Store | Bin 8196 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index a63eb67cdcbd46fe107cfc6eabe3a27ef312fe18..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHM-D(p-6h70Y$%eL4sA5or1#fGOZ7mdU(likY{y?|rg-T4CXqRqx;%?F?A&_f* z2VcQkpTu|ZO20ERExVf*T98(BCd{14%=ymDd~;?uXF^1x+HXG~DiM)`%yMBC#VLvV zIZvc1)3OLDfG6rvydB8BRv=pw+P1+mU>UFsSOzQumVv*40lc$0IcL20b+0Wg1D1jR zk^z1`xX3IkMvk>qj}815(}K1!R&%fD1o;9F?v zQ=5E}bdPo^0yYG84>g~O`c z$ymL4ZmT{$CvVc4xRzM=1+BFPao5~}d~bCFKkk*wKg85@ZYE!F3eK$a#@m+zuh;DL z<7zW{#ou;X!g{*HDLTc* zLTNZ$S$(+Vu5GN1mfYdSN*VRl_0g#4+_-!H(N5zaI1J@ywVx!+1C{3W_mlDqY8PDf zkr#!54BrFER`c)^3C}pdjzt-M*?Q8iaz;0Ke*|9HK)8M`E=JdVRu@5J4^T73?*-+;wq9a~2 zM(on3^oYt&lgL8q^kmo1JSDXqb58rPe42IUE2+PTGs-K7u@e#4*u@M7IGH|Lx@L|O ztSBCnR%XCJ2Z%`?z+dW|%w?=MD8N`?Bh3Oo1I~xL`jfQ&Cr;9tykyP{w$CQZz@;!S ztw$+w{9ix&{{K?uVh6DdSO%_!0g^t1tT)zDK~Si098!hj kkSBi_qV7VKF%=`nTA~H%(k}uo{7c%t|J6p>;-6yRHyWv(_5c6?