Swift - Add watermark to a video is very slow

Question

Here is my code that adds image & text overlays to a local video. The problem is that it's extremely SLOW. Any ideas how to fix it? Also I would appreciate if you can suggest 3rd party libraries that can do watermarking.

public func addWatermark(
    fromVideoAt videoURL: URL,
    watermark: Watermark,
    fileName: String,
    onSuccess: @escaping (URL) -> Void,
    onFailure: @escaping ((Error?) -> Void)
) {
    let asset = AVURLAsset(url: videoURL)
    let composition = AVMutableComposition()

    guard
        let compositionTrack = composition.addMutableTrack(
            withMediaType: .video,
            preferredTrackID: kCMPersistentTrackID_Invalid
        ),
        let assetTrack = asset.tracks(withMediaType: .video).first
    else {
        onFailure(nil)
        return
    }

    do {
        let timeRange = CMTimeRange(start: .zero, duration: assetTrack.timeRange.duration)
        try compositionTrack.insertTimeRange(timeRange, of: assetTrack, at: .zero)

        if let audioAssetTrack = asset.tracks(withMediaType: .audio).first,
           let compositionAudioTrack = composition.addMutableTrack(
               withMediaType: .audio,
               preferredTrackID: kCMPersistentTrackID_Invalid
           ) {
            try compositionAudioTrack.insertTimeRange(
                timeRange,
                of: audioAssetTrack,
                at: .zero
            )
        }
    } catch {
        onFailure(error)
        return
    }

    compositionTrack.preferredTransform = assetTrack.preferredTransform
    let videoInfo = orientation(from: assetTrack.preferredTransform)

    let videoSize: CGSize
    if videoInfo.isPortrait {
        videoSize = CGSize(
            width: assetTrack.naturalSize.height,
            height: assetTrack.naturalSize.width
        )
    } else {
        videoSize = assetTrack.naturalSize
    }

    let videoLayer = CALayer()
    videoLayer.frame = CGRect(origin: .zero, size: videoSize)
    let overlayLayer = CALayer()
    overlayLayer.frame = CGRect(origin: .zero, size: videoSize)

    videoLayer.frame = CGRect(x: 0, y: 0, width: videoSize.width, height: videoSize.height)

    let imageFrame = watermark.calculateImageFrame(parentSize: videoSize)
    addImage(watermark.image, to: overlayLayer, frame: imageFrame)
    let textOrigin = CGPoint(x: imageFrame.minX + 4, y: imageFrame.minY)
    if let text = watermark.text {
        addText(
            text,
            to: overlayLayer,
            origin: textOrigin,
            textAttributes: Watermark.textAttributes(type: watermark.type)
        )
    }

    let outputLayer = CALayer()
    outputLayer.frame = CGRect(origin: .zero, size: videoSize)
    outputLayer.addSublayer(videoLayer)
    outputLayer.addSublayer(overlayLayer)

    let videoComposition = AVMutableVideoComposition()
    videoComposition.renderSize = videoSize
    videoComposition.frameDuration = CMTime(value: 1, timescale: 60)
    videoComposition.animationTool = AVVideoCompositionCoreAnimationTool(
        postProcessingAsVideoLayer: videoLayer,
        in: outputLayer
    )
    videoComposition.colorPrimaries = AVVideoColorPrimaries_ITU_R_709_2
    videoComposition.colorTransferFunction = "sRGB"
    videoComposition.colorYCbCrMatrix = nil

    let instruction = AVMutableVideoCompositionInstruction()
    instruction.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
    videoComposition.instructions = [instruction]
    let layerInstruction = compositionLayerInstruction(
        for: compositionTrack,
        assetTrack: assetTrack
    )
    instruction.layerInstructions = [layerInstruction]

    guard let export = AVAssetExportSession(
        asset: composition,
        presetName: AVAssetExportPresetHighestQuality
    )
    else {
        onFailure(nil)
        return
    }

    let exportURL = URL(fileURLWithPath: NSTemporaryDirectory())
        .appendingPathComponent(fileName)
        .appendingPathExtension("mov")

    export.videoComposition = videoComposition
    export.outputFileType = .mov
    export.outputURL = exportURL

    export.exportAsynchronously {
        DispatchQueue.main.async {
            switch export.status {
            case .completed:
                onSuccess(exportURL)
            default:
                onFailure(export.error)
            }
        }
    }
}

Watermark is the wrapper struct. It contains image/text, text attributes, size and other similar helpful information.

I've tried without any luck:

• export.shouldOptimizeForNetworkUse = true. It did not work.
• AVAssetExportPresetPassthrough instead of AVAssetExportPresetHighestQuality. It removed overlays.

Answer 1

I have the following code which is relatively fast. It watermarks an 8 second video in about 2.56 seconds. When I ran it under Metal System Trace Instrument it seemed to be balanced and using GPU-acceleration the whole time. You just call exportIt()

As a side matter, this code uses async await wrapping of AVKit functions and migrates off any deprecated interfaces as of iOS 16.

A tidied up and working sample app with resource files is https://github.com/faisalmemon/watermark

The core code is as follows:

//
//  WatermarkHelper.swift
//  watermark
//
//  Created by Faisal Memon on 09/02/2023.
//

import Foundation
import AVKit

struct WatermarkHelper {
    
    enum WatermarkError: Error {
        case cannotLoadResources
        case cannotAddTrack
        case cannotLoadVideoTrack(Error?)
        case cannotCopyOriginalAudioVideo(Error?)
        case noVideoTrackPresent
        case exportSessionCannotBeCreated
    }
   
    func compositionAddMediaTrack(_ composition: AVMutableComposition, withMediaType mediaType: AVMediaType) throws -> AVMutableCompositionTrack  {
        guard let compositionTrack = composition.addMutableTrack(
            withMediaType: mediaType,
            preferredTrackID: kCMPersistentTrackID_Invalid) else {
            throw WatermarkError.cannotAddTrack
        }
        return compositionTrack
    }
    
    func loadTrack(inputVideo: AVAsset, withMediaType mediaType: AVMediaType) async throws -> AVAssetTrack? {
        return try await withCheckedThrowingContinuation({
            (continuation: CheckedContinuation<AVAssetTrack?, Error>) in
            
            inputVideo.loadTracks(withMediaType: mediaType) { tracks, error in
                if let tracks = tracks {
                    continuation.resume(returning: tracks.first)
                } else {
                    continuation.resume(throwing: WatermarkError.cannotLoadVideoTrack(error))
                }
            }
        })
    }
    
    func bringOverVideoAndAudio(inputVideo: AVAsset, assetTrack: AVAssetTrack, compositionTrack: AVMutableCompositionTrack, composition: AVMutableComposition) async throws {
        do {
            let timeRange = await CMTimeRange(start: .zero, duration: try inputVideo.load(.duration))
            try compositionTrack.insertTimeRange(timeRange, of: assetTrack, at: .zero)
            if let audioAssetTrack = try await loadTrack(inputVideo: inputVideo, withMediaType: .audio) {
                let compositionAudioTrack = try compositionAddMediaTrack(composition, withMediaType: .audio)
                try compositionAudioTrack.insertTimeRange(timeRange, of: audioAssetTrack, at: .zero)
            }
        } catch {
            print(error)
            throw WatermarkError.cannotCopyOriginalAudioVideo(error)
        }
    }
    
    private func orientation(from transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
        var assetOrientation = UIImage.Orientation.up
        var isPortrait = false
        if transform.a == 0 && transform.b == 1.0 && transform.c == -1.0 && transform.d == 0 {
            assetOrientation = .right
            isPortrait = true
        } else if transform.a == 0 && transform.b == -1.0 && transform.c == 1.0 && transform.d == 0 {
            assetOrientation = .left
            isPortrait = true
        } else if transform.a == 1.0 && transform.b == 0 && transform.c == 0 && transform.d == 1.0 {
            assetOrientation = .up
        } else if transform.a == -1.0 && transform.b == 0 && transform.c == 0 && transform.d == -1.0 {
            assetOrientation = .down
        }
        
        return (assetOrientation, isPortrait)
    }
    
    func preferredTransformAndSize(compositionTrack: AVMutableCompositionTrack, assetTrack: AVAssetTrack) async throws -> (preferredTransform: CGAffineTransform, videoSize: CGSize) {
        
        let transform = try await assetTrack.load(.preferredTransform)
        let videoInfo = orientation(from: transform)
        
        let videoSize: CGSize
        let naturalSize = try await assetTrack.load(.naturalSize)
        if videoInfo.isPortrait {
            videoSize = CGSize(
                width: naturalSize.height,
                height: naturalSize.width)
        } else {
            videoSize = naturalSize
        }
        return (transform, videoSize)
    }
    
    private func compositionLayerInstruction(for track: AVCompositionTrack, assetTrack: AVAssetTrack, preferredTransform: CGAffineTransform) -> AVMutableVideoCompositionLayerInstruction {
        
        let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: track)
        instruction.setTransform(preferredTransform, at: .zero)
        
        return instruction
    }
    
    private func addImage(to layer: CALayer, watermark: UIImage, videoSize: CGSize) {
        let imageLayer = CALayer()
        let aspect: CGFloat = watermark.size.width / watermark.size.height
        let width = videoSize.width
        let height = width / aspect
        imageLayer.frame = CGRect(
            x: 0,
            y: -height * 0.15,
            width: width,
            height: height)
        imageLayer.contents = watermark.cgImage
        layer.addSublayer(imageLayer)
    }

    
    func composeVideo(composition: AVMutableComposition, videoComposition: AVMutableVideoComposition, compositionTrack: AVMutableCompositionTrack, assetTrack: AVAssetTrack, preferredTransform: CGAffineTransform) {
        
        let instruction = AVMutableVideoCompositionInstruction()
        instruction.timeRange = CMTimeRange(
            start: .zero,
            duration: composition.duration)
        videoComposition.instructions = [instruction]
        let layerInstruction = compositionLayerInstruction(
            for: compositionTrack,
            assetTrack: assetTrack, preferredTransform: preferredTransform)
        instruction.layerInstructions = [layerInstruction]
    }
    
    func exportSession(composition: AVMutableComposition, videoComposition: AVMutableVideoComposition, outputURL: URL) throws -> AVAssetExportSession {
        guard let export = AVAssetExportSession(
          asset: composition,
          presetName: AVAssetExportPresetHighestQuality)
          else {
            print("Cannot create export session.")
            throw WatermarkError.exportSessionCannotBeCreated
        }
        export.videoComposition = videoComposition
        export.outputFileType = .mp4
        export.outputURL = outputURL
        return export
    }
    
    func executeSession(_ session: AVAssetExportSession) async throws -> AVAssetExportSession.Status {

        return try await withCheckedThrowingContinuation({
            (continuation: CheckedContinuation<AVAssetExportSession.Status, Error>) in
            session.exportAsynchronously {
                DispatchQueue.main.async {
                    if let error = session.error {
                        continuation.resume(throwing: error)
                    } else {
                        continuation.resume(returning: session.status)
                    }
                }
            }
        })
    }
    
    func addWatermarkTopDriver(inputVideo: AVAsset, outputURL: URL, watermark: UIImage) async throws -> AVAssetExportSession.Status {
        let composition = AVMutableComposition()
        let compositionTrack = try compositionAddMediaTrack(composition, withMediaType: .video)
        guard let videoAssetTrack = try await loadTrack(inputVideo: inputVideo, withMediaType: .video) else {
            throw WatermarkError.noVideoTrackPresent
        }
        try await bringOverVideoAndAudio(inputVideo: inputVideo, assetTrack: videoAssetTrack, compositionTrack: compositionTrack, composition: composition)
        let transformAndSize = try await preferredTransformAndSize(compositionTrack: compositionTrack, assetTrack: videoAssetTrack)
        compositionTrack.preferredTransform = transformAndSize.preferredTransform
        
        let videoLayer = CALayer()
        videoLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
        let overlayLayer = CALayer()
        overlayLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
        addImage(to: overlayLayer, watermark: watermark, videoSize: transformAndSize.videoSize)

        let outputLayer = CALayer()
        outputLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
        outputLayer.addSublayer(videoLayer)
        outputLayer.addSublayer(overlayLayer)
        
        let videoComposition = AVMutableVideoComposition()
        videoComposition.renderSize = transformAndSize.videoSize
        videoComposition.frameDuration = CMTime(value: 1, timescale: 30)
        videoComposition.animationTool = AVVideoCompositionCoreAnimationTool(
            postProcessingAsVideoLayer: videoLayer,
            in: outputLayer)
        composeVideo(composition: composition, videoComposition: videoComposition, compositionTrack: compositionTrack, assetTrack: videoAssetTrack, preferredTransform: transformAndSize.preferredTransform)
        
        let session = try exportSession(composition: composition, videoComposition: videoComposition, outputURL: outputURL)
        return try await executeSession(session)
    }
    
    /// Creates a watermarked movie and saves it to the documents directory.
    ///
    /// For an 8 second video (251 frames), this code takes 2.56 seconds on iPhone 11 producing a high quality video at 30 FPS.
    /// - Returns: Time interval taken for processing.
    public func exportIt() async throws -> TimeInterval {
        let timeStart = Date()
        guard
            let filePath = Bundle.main.path(forResource: "donut-spinning", ofType: "mp4"),
            let docUrl = try? FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true),
            let watermarkImage = UIImage(systemName: "seal") else {
            throw WatermarkError.cannotLoadResources
        }
        let videoAsset = AVAsset(url: URL(filePath: filePath))
        
        let outputURL = docUrl.appending(component: "watermark-donut-spinning.mp4")
        try? FileManager.default.removeItem(at: outputURL)
        print(outputURL)
        let result = try await addWatermarkTopDriver(inputVideo: videoAsset, outputURL: outputURL, watermark: watermarkImage)
        let timeEnd = Date()
        let duration = timeEnd.timeIntervalSince(timeStart)
        print(result)
        return duration
    }
}

Answer 2

Have you checked out Apple's documentation? It adds a title layer (CALayer) on top of an existing AVMutableComposition or an AVAsset? Since it's a legacy doc from iOS 6, you'll need to refactor a bit, but it should be fast on today's tech.

Answer 3

Use this below method for super fast watermark adding to video

func addWatermark(inputURL: URL, outputURL: URL, handler:@escaping (_ exportSession: AVAssetExportSession?)-> Void) {
    let mixComposition = AVMutableComposition()
    let asset = AVAsset(url: inputURL)
    let videoTrack = asset.tracks(withMediaType: AVMediaType.video)[0]
    let timerange = CMTimeRangeMake(start: CMTime.zero, duration: asset.duration)

        let compositionVideoTrack:AVMutableCompositionTrack = mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: CMPersistentTrackID(kCMPersistentTrackID_Invalid))!

    do {
        try compositionVideoTrack.insertTimeRange(timerange, of: videoTrack, at: CMTime.zero)
        compositionVideoTrack.preferredTransform = videoTrack.preferredTransform
    } catch {
        print(error)
    }

    let watermarkFilter = CIFilter(name: "CISourceOverCompositing")!
    let watermarkImage = CIImage(image: UIImage(named: "waterMark")!)
    let videoComposition = AVVideoComposition(asset: asset) { (filteringRequest) in
        let source = filteringRequest.sourceImage.clampedToExtent()
        watermarkFilter.setValue(source, forKey: "inputBackgroundImage")
        let transform = CGAffineTransform(translationX: filteringRequest.sourceImage.extent.width - (watermarkImage?.extent.width)! - 2, y: 0)
        watermarkFilter.setValue(watermarkImage?.transformed(by: transform), forKey: "inputImage")
        filteringRequest.finish(with: watermarkFilter.outputImage!, context: nil)
    }

    guard let exportSession = AVAssetExportSession(asset: asset, presetName: AVAssetExportPreset640x480) else {
        handler(nil)

        return
    }

    exportSession.outputURL = outputURL
    exportSession.outputFileType = AVFileType.mp4
    exportSession.shouldOptimizeForNetworkUse = true
    exportSession.videoComposition = videoComposition
    exportSession.exportAsynchronously { () -> Void in
        handler(exportSession)
    }
}

Call this method when you want to add watermark easily

let outputURL = NSURL.fileURL(withPath: "TempPath")
let inputURL = NSURL.fileURL(withPath: "VideoWithWatermarkPath")
addWatermark(inputURL: inputURL, outputURL: outputURL, handler: { (exportSession) in
    guard let session = exportSession else {
        // Error 
        return
    }
    switch session.status {
        case .completed:
        guard NSData(contentsOf: outputURL) != nil else {
            // Error
            return
        }

        // Now you can find the video with the watermark in the location outputURL

        default:
        // Error
    }
})