Seth Haberman
Reputation: 141
NDI Audio playback on iOS simulator is great but iPad is glitchy and stuttery
I am trying to implement an NDI audio receiver on iOS. It sounds great on the iOS simulator but on the iPad (iPad Air 2?) there is some stuttering and glitching. I am pretty sure it has to do with the buffer playback but am not sure how to fix it or what is wrong.
Here is my NDI manager class for the playback
import Foundation
import AVFoundation
class NDIManager {
static let shared = NDIManager()
private init() {}
private var ndiFinderInstance: OpaquePointer?
private var ndiReceiverInstance: OpaquePointer?
private var audioEngine = AVAudioEngine()
private var playerNode = AVAudioPlayerNode()
private var format: AVAudioFormat!
private var buffers: [AVAudioPCMBuffer] = []
private var isPlayerStarted = false
private let bufferQueue = DispatchQueue(label: "com.ndimanager.bufferQueue")
private var lastResetTime: TimeInterval = 0
private let bufferQueueLimit = 50 // Increase limit for smoother playback
// Setup NDI Finder
func setupNDIFinder() -> Bool {
cleanupFinder()
var finderConfig = NDIlib_find_create_t()
finderConfig.show_local_sources = true
ndiFinderInstance = NDIlib_find_create_v2(&finderConfig)
return ndiFinderInstance != nil
}
// List available NDI sources
func listNDISources() -> [NDIlib_source_t] {
guard let finder = ndiFinderInstance else { return [] }
var sourceCount: UInt32 = 0
let sourcesPointer = NDIlib_find_get_current_sources(finder, &sourceCount)
guard let sources = sourcesPointer else { return [] }
return Array(UnsafeBufferPointer(start: sources, count: Int(sourceCount)))
}
// Automatically connect to "vMix Audio - Headphones" source if available
public func connectToHeadphones() {
if !setupNDIFinder() {
print("Failed to set up NDI finder.")
return
}
let targetName = "vMix Audio - Headphones"
DispatchQueue.global().async {
while true {
let sources = self.listNDISources()
if sources.isEmpty {
print("No NDI sources found. Retrying...")
sleep(1)
continue
}
for source in sources {
let sourceName = String(cString: source.p_ndi_name)
print("Available source: \(sourceName)")
if sourceName.contains(targetName) {
DispatchQueue.main.async {
print("Found matching NDI source: \(sourceName)")
if self.setupNDIReceiver(withSource: source) {
print("Successfully connected to NDI source: \(sourceName)")
} else {
print("Failed to connect to NDI source: \(sourceName)")
}
}
return
}
}
print("No NDI source with name containing '\(targetName)' found. Retrying...")
sleep(1)
}
}
}
// Setup NDI receiver with the given source
private func setupNDIReceiver(withSource source: NDIlib_source_t) -> Bool {
cleanupReceiver()
var recvConfig = NDIlib_recv_create_v3_t()
recvConfig.source_to_connect_to = source
recvConfig.color_format = NDIlib_recv_color_format_BGRX_BGRA
recvConfig.bandwidth = NDIlib_recv_bandwidth_highest
recvConfig.allow_video_fields = true
ndiReceiverInstance = NDIlib_recv_create_v3(&recvConfig)
guard ndiReceiverInstance != nil else {
print("Failed to create NDI receiver.")
return false
}
setupAudioEngine()
DispatchQueue.global().async {
self.collectAndPlayAudioFrames()
}
return true
}
func setupAudioEngine() {
format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false)
audioEngine.attach(playerNode)
audioEngine.connect(playerNode, to: audioEngine.mainMixerNode, format: format)
do {
try audioEngine.start()
print("Audio engine started.")
} catch {
print("Audio engine failed to start: \(error)")
}
}
private func collectAndPlayAudioFrames() {
guard let receiver = ndiReceiverInstance else {
print("NDI receiver instance is nil.")
return
}
while true {
var audioFrame = NDIlib_audio_frame_v2_t()
let captureResult = NDIlib_recv_capture_v2(receiver, nil, &audioFrame, nil, 5000)
if captureResult == NDIlib_frame_type_audio {
let pcmData = Data(bytes: audioFrame.p_data!, count: Int(audioFrame.no_samples * audioFrame.no_channels) * MemoryLayout<Float>.size)
streamPCMChunk(pcmData, isFirst: buffers.isEmpty)
NDIlib_recv_free_audio_v2(receiver, &audioFrame)
}
}
}
func streamPCMChunk(_ floatPCMData: Data, isFirst: Bool = false) {
bufferQueue.async { [weak self] in
guard let self = self else { return }
// Reset buffers only when absolutely necessary
let currentTime = Date().timeIntervalSince1970
if isFirst && !self.isPlayerStarted && (currentTime - self.lastResetTime > 2.0) {
self.resetBuffers()
self.lastResetTime = currentTime
}
guard let buffer = self.createPCMBuffer(from: floatPCMData) else {
print("Failed to create PCM buffer")
return
}
// Add buffer to queue if below the limit
if self.buffers.count < self.bufferQueueLimit {
self.buffers.append(buffer)
} else {
print("Buffer queue limit reached; dropping buffer.")
}
if !self.isPlayerStarted {
print("Starting playback.")
self.playerNode.play()
self.isPlayerStarted = true
}
if self.buffers.count == 1 {
self.scheduleNextBuffer()
}
}
}
private func scheduleNextBuffer() {
bufferQueue.async { [weak self] in
guard let self = self else { return }
guard !self.buffers.isEmpty else {
self.playerNode.stop()
self.isPlayerStarted = false
return
}
let buffer = self.buffers.removeFirst()
self.playerNode.scheduleBuffer(buffer) {
self.scheduleNextBuffer()
}
}
}
private func createPCMBuffer(from data: Data) -> AVAudioPCMBuffer? {
let frameCount = data.count / MemoryLayout<Float>.size / Int(format.channelCount)
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else {
return nil
}
buffer.frameLength = AVAudioFrameCount(frameCount)
data.withUnsafeBytes { floatBytes in
if let floatPointer = floatBytes.baseAddress?.assumingMemoryBound(to: Float.self) {
buffer.floatChannelData?[0].assign(from: floatPointer, count: frameCount)
buffer.floatChannelData?[1].assign(from: floatPointer.advanced(by: frameCount), count: frameCount)
}
}
return buffer
}
private func resetBuffers() {
bufferQueue.async { [weak self] in
self?.buffers.removeAll()
self?.playerNode.stop()
self?.playerNode.reset()
self?.isPlayerStarted = false
}
}
func cleanupFinder() {
if let finder = ndiFinderInstance {
NDIlib_find_destroy(finder)
ndiFinderInstance = nil
}
}
func cleanupReceiver() {
if let receiver = ndiReceiverInstance {
NDIlib_recv_destroy(receiver)
ndiReceiverInstance = nil
}
audioEngine.stop()
playerNode.stop()
isPlayerStarted = false
}
}
This is some console for the iOS simulator
AddInstanceForFactory: No factory registered for id <CFUUID 0x60000262c8e0> F8BB1C28-BAE8-11D6-9C31-00039315CD46
AudioConverter.cpp:1007 Failed to create a new in process converter -> from 1 ch, 48000 Hz, Float32 to 0 ch, 0 Hz, with status -50
AudioConverter.cpp:1007 Failed to create a new in process converter -> from 1 ch, 48000 Hz, Float32 to 0 ch, 0 Hz, with status -50
AudioConverter.cpp:1007 Failed to create a new in process converter -> from 1 ch, 48000 Hz, Float32 to 0 ch, 0 Hz, with status -50
AudioConverter.cpp:1007 Failed to create a new in process converter -> from 1 ch, 48000 Hz, Float32 to 0 ch, 0 Hz, with status -50
AQMEIO_HAL.cpp:893 kAudioDevicePropertyMute returned err 2003332927
Audio engine started.
Successfully connected to NDI source: JUDITHHABERF36A (vMix Audio - Headphones)
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Buffers reset and playerNode stopped.
Starting playerNode playback.
Buffer is empty; stopping playback.
Finished playing buffer. Scheduling next buffer.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Buffers reset and playerNode stopped.
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Requesting XML data from vMix...
Not connected or output stream unavailable
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Here is the console from the physical iPad
Found matching NDI source: JUDITHHABERF36A (vMix Audio - Headphones)
Audio engine started.
Successfully connected to NDI source: JUDITHHABERF36A (vMix Audio - Headphones)
Requesting XML data from vMix...
Not connected or output stream unavailable
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Buffers reset and playerNode stopped.
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Buffer is empty; stopping playback.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Finished playing buffer. Scheduling next buffer.
Buffer is empty; stopping playback.
Buffer is empty; stopping playback.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Starting playerNode playback.
Buffers reset and playerNode stopped.
Buffer is empty; stopping playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
Starting playerNode playback.
Audio frame received: sample_rate = 48000, channels = 2, samples = 960
UPDATE That suggestion worked! Here is the updated buffer and stream code
func streamPCMChunk(_ floatPCMData: Data, isFirst: Bool = false) {
bufferQueue.async { [weak self] in
guard let self = self else { return }
// Reset buffers only once at the start of the stream
if isFirst && !self.isPlayerStarted {
self.resetBuffers()
}
// Create PCM buffer
guard let buffer = self.createPCMBuffer(from: floatPCMData) else {
print("Failed to create PCM buffer")
return
}
self.buffers.append(buffer)
// Start playback when enough buffers are preloaded
if !self.isPlayerStarted && self.buffers.count >= self.buffersToScheduleAtOnce {
self.playerNode.play()
self.isPlayerStarted = true
print("Player started with \(self.buffers.count) preloaded buffers.")
}
// Schedule buffers if the player is active
if self.isPlayerStarted {
self.scheduleNextBuffer()
}
}
}
private func scheduleNextBuffer() {
bufferQueue.async { [weak self] in
guard let self = self else { return }
// Ensure there are buffers to schedule
guard !self.buffers.isEmpty else {
print("Buffer underflow: no buffers to schedule.")
return
}
// Remove excess buffers if the queue is too large
if self.buffers.count > self.maxBufferQueueSize {
let excess = self.buffers.count - self.maxBufferQueueSize
self.buffers.removeFirst(excess)
print("Removed \(excess) excess buffers. Remaining queue length: \(self.buffers.count)")
}
// Schedule multiple buffers
let buffersToSchedule = min(self.buffersToScheduleAtOnce, self.buffers.count)
for _ in 0..<buffersToSchedule {
let buffer = self.buffers.removeFirst()
self.playerNode.scheduleBuffer(buffer) {
self.bufferQueue.async {
self.scheduleNextBuffer()
}
}
}
print("Scheduled \(buffersToSchedule) buffers for playback. Remaining queue length: \(self.buffers.count)")
}
}
private func createPCMBuffer(from data: Data) -> AVAudioPCMBuffer? {
let frameCount = data.count / MemoryLayout<Float>.size / Int(format.channelCount)
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else {
return nil
}
buffer.frameLength = AVAudioFrameCount(frameCount)
data.withUnsafeBytes { floatBytes in
if let floatPointer = floatBytes.baseAddress?.assumingMemoryBound(to: Float.self) {
buffer.floatChannelData?[0].assign(from: floatPointer, count: frameCount)
buffer.floatChannelData?[1].assign(from: floatPointer.advanced(by: frameCount), count: frameCount)
}
}
return buffer
}
Upvotes: 1
Views: 47
Answers (1)
Gordon Childs
Reputation: 36072
You're getting audio dropouts because you're playing too "close" (one buffer's worth) to the end of what is available and the player is running out of audio data.
Instead, wait until you have more audio, e.g. a second, and then schedule more, perhaps even all of it.
This is known as buffering. If you've ever watched a video online, you'll have experienced buffering & perhaps even seen the buffer size in that small bar that extends beyond the current playback position.
Upvotes: 0
Related Questions
- Xcode building for iOS Simulator, but linking in an object file built for iOS, for architecture 'arm64'
- How to use dark mode in iOS simulator?
- Take screenshots in the iOS simulator
- Is it possible to disable the network in iOS Simulator?
- Capture iOS Simulator video for App Preview
- iOS detect if user is on an iPad
- Building for iOS Simulator, but the linked framework '****.framework' was built for iOS
- Fast and Lean PDF Viewer for iPhone / iPad / iOS - tips and hints?