Tap audio output using AVAudioEngine

Question

I'm trying install a tap on the output audio that is played on my app. I have no issue catching buffer from microphone input, but when it comes to catch sound that it goes trough the speaker or the earpiece or whatever the output device is, it does not succeed. Am I missing something?

In my example I'm trying to catch the audio buffer from an audio file that an AVPLayer is playing. But let's pretend I don't have access directly to the AVPlayer instance.

The goal is to perform Speech Recognition on an audio stream.

func catchAudioBuffers() throws {
    let audioSession = AVAudioSession.sharedInstance()
    try audioSession.setCategory(.playAndRecord, mode: .voiceChat, options: .allowBluetooth)
    try audioSession.setActive(true)

    let outputNode = audioEngine.outputNode

    let recordingFormat = outputNode.outputFormat(forBus: 0)
    outputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { (buffer: AVAudioPCMBuffer, when: AVAudioTime) in
      // PROCESS AUDIO BUFFER
    }

    audioEngine.prepare()
    try audioEngine.start()

    // For example I am playing an audio conversation with an AVPlayer and a local file.
    player.playSound()
}

This code results in a:

AVAEInternal.h:76    required condition is false: [AVAudioIONodeImpl.mm:1057:SetOutputFormat: (_isInput)]
*** Terminating app due to uncaught exception 'com.apple.coreaudio.avfaudio', reason: 'required condition is false: _isInput'

Answer 1

I was facing the same problem and during 2 days of brainstorming found the following.

Apple says that For AVAudioOutputNode, tap format must be specified as nil. I'm not sure that it's important but in my case, that finally worked, format was nil. You need to start recording and don't forget to stop it.

Removing tap is really important, otherwise you will have file that you can't open.

Try to save the file with the same audio settings that you used in source file.

Here's my code that finally worked. It was partly taken from this question Saving Audio After Effect in iOS.

    func playSound() {
    let rate: Float? = effect.speed
    let pitch: Float? = effect.pitch
    let echo: Bool? = effect.echo
    let reverb: Bool? = effect.reverb

    // initialize audio engine components
    audioEngine = AVAudioEngine()

    // node for playing audio
    audioPlayerNode = AVAudioPlayerNode()
    audioEngine.attach(audioPlayerNode)

    // node for adjusting rate/pitch
    let changeRatePitchNode = AVAudioUnitTimePitch()
    if let pitch = pitch {
        changeRatePitchNode.pitch = pitch
    }
    if let rate = rate {
        changeRatePitchNode.rate = rate
    }
    audioEngine.attach(changeRatePitchNode)

    // node for echo
    let echoNode = AVAudioUnitDistortion()
    echoNode.loadFactoryPreset(.multiEcho1)
    audioEngine.attach(echoNode)

    // node for reverb
    let reverbNode = AVAudioUnitReverb()
    reverbNode.loadFactoryPreset(.cathedral)
    reverbNode.wetDryMix = 50
    audioEngine.attach(reverbNode)

    // connect nodes
    if echo == true && reverb == true {
        connectAudioNodes(audioPlayerNode, changeRatePitchNode, echoNode, reverbNode, audioEngine.mainMixerNode, audioEngine.outputNode)
    } else if echo == true {
        connectAudioNodes(audioPlayerNode, changeRatePitchNode, echoNode, audioEngine.mainMixerNode, audioEngine.outputNode)
    } else if reverb == true {
        connectAudioNodes(audioPlayerNode, changeRatePitchNode, reverbNode, audioEngine.mainMixerNode, audioEngine.outputNode)
    } else {
        connectAudioNodes(audioPlayerNode, changeRatePitchNode, audioEngine.mainMixerNode, audioEngine.outputNode)
    }

    // schedule to play and start the engine!
    audioPlayerNode.stop()
    audioPlayerNode.scheduleFile(audioFile, at: nil) {
        var delayInSeconds: Double = 0
        if let lastRenderTime = self.audioPlayerNode.lastRenderTime, let playerTime = self.audioPlayerNode.playerTime(forNodeTime: lastRenderTime) {
            if let rate = rate {
                delayInSeconds = Double(self.audioFile.length - playerTime.sampleTime) / Double(self.audioFile.processingFormat.sampleRate) / Double(rate)
            } else {
                delayInSeconds = Double(self.audioFile.length - playerTime.sampleTime) / Double(self.audioFile.processingFormat.sampleRate)
            }
        }

        // schedule a stop timer for when audio finishes playing
        self.stopTimer = Timer(timeInterval: delayInSeconds, target: self, selector: #selector(EditViewController.stopAudio), userInfo: nil, repeats: false)
        RunLoop.main.add(self.stopTimer!, forMode: RunLoop.Mode.default)
    }

    do {
        try audioEngine.start()
    } catch {
        showAlert(Alerts.AudioEngineError, message: String(describing: error))
        return
    }

    //Try to save
    let dirPaths: String = (NSSearchPathForDirectoriesInDomains(.libraryDirectory, .userDomainMask, true)[0]) + "/sounds/"
    let tmpFileUrl = URL(fileURLWithPath: dirPaths + "effected.caf")

    //Save the tmpFileUrl into global varibale to not lose it (not important if you want to do something else)
    filteredOutputURL = URL(fileURLWithPath: filePath)
        do{
            print(dirPaths)
            let settings = [AVSampleRateKey : NSNumber(value: Float(44100.0)),
            AVFormatIDKey : NSNumber(value: Int32(kAudioFormatMPEG4AAC)),
            AVNumberOfChannelsKey : NSNumber(value: 1),
            AVEncoderAudioQualityKey : NSNumber(value: Int32(AVAudioQuality.medium.rawValue))]
            self.newAudio = try! AVAudioFile(forWriting: tmpFileUrl as URL, settings: settings)
            let length = self.audioFile.length
            audioEngine.mainMixerNode.installTap(onBus: 0, bufferSize: 4096, format: nil) {
                (buffer: AVAudioPCMBuffer?, time: AVAudioTime!) -> Void in
                //Let us know when to stop saving the file, otherwise saving infinitely
                if (self.newAudio.length) <= length {
                    do{
                        try self.newAudio.write(from: buffer!)

                    } catch _{
                        print("Problem Writing Buffer")
                    }
                } else {
                    //if we dont remove it, will keep on tapping infinitely
                    self.audioEngine.mainMixerNode.removeTap(onBus: 0)
                }
            }
    }

    // play the recording!
    audioPlayerNode.play()

}

@objc func stopAudio() {
    if let audioPlayerNode = audioPlayerNode {
        let engine = audioEngine
        audioPlayerNode.stop()
        engine?.mainMixerNode.removeTap(onBus: 0)

    }
    if let stopTimer = stopTimer {
        stopTimer.invalidate()
    }
    configureUI(.notPlaying)
    if let audioEngine = audioEngine {
        audioEngine.stop()
        audioEngine.reset()
    }
    isPlaying = false
}