converting image to binary in swift

Question

I want to convert an image to binary black and white, at the moment I am looping through pixels (stored in UnsafeMutableBufferPointer) using normal nested loops, comparing each RGB to average and setting it to black or white, however.

This seems really slow and I am sure there is a built in a way that uses gpu or is well optimized. If you could provide a code sample or link it'd be great.

for var y in 0..<height {
    for var x in 0..<width{
        //Pixel is small class i made for 8 bit access and comparison
        if (Buffer[x+y*width]  < AVRRGB) {
            Buffer[x+y*width] = Pixel(RGB: 0x000000FF)
        } else{
            Buffer[x+y*width] = Pixel(RGB: 0xFFFFFFFF)
        }
    }
}

Answer 1

A couple of observations:

1. Make sure you're doing a test on a device with a release build (or optimizations turned off). That alone makes it much faster. On iPhone 7+ it reduced the conversion of 1920 x 1080 pixel color image to grayscale from 1.7 seconds to less than 0.1 seconds.

2. You might want to use DispatchQueue.concurrentPerform to process pixels concurrently. On my iPhone 7+, that made it about twice as fast.

In my experience Core Image filters weren't much faster, but you can contemplate vImage or Metal if you need it much faster. But unless you're dealing with extraordinarily large images, the response time with optimized (and possibly concurrent) simple Swift code might be sufficient.

An unrelated observation:

3. Also, I'm not sure how your conversion to black and white works, but often you'd want to calculate the relative luminance of the color pixel (e.g. 0.2126 * red + 0.7152 * green + 0.0722 * blue). Certainly when converting color image to grayscale you'd do something like that to get an image that more closely represents what the human eye can see, and I'd personally do something like that if converting to black and white, too.

---

FYI, my Swift 3/4 color-to-grayscale routine looks like:

func blackAndWhite(image: UIImage, completion: @escaping (UIImage?) -> Void) {
    DispatchQueue.global(qos: .userInitiated).async {
        // get information about image

        let imageref = image.cgImage!
        let width = imageref.width
        let height = imageref.height

        // create new bitmap context

        let bitsPerComponent = 8
        let bytesPerPixel = 4
        let bytesPerRow = width * bytesPerPixel
        let colorSpace = CGColorSpaceCreateDeviceRGB()
        let bitmapInfo = Pixel.bitmapInfo
        let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: bitsPerComponent, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo)!

        // draw image to context

        let rect = CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height))
        context.draw(imageref, in: rect)

        // manipulate binary data

        guard let buffer = context.data else {
            print("unable to get context data")
            completion(nil)
            return
        }

        let pixels = buffer.bindMemory(to: Pixel.self, capacity: width * height)

        DispatchQueue.concurrentPerform(iterations: height) { row in
            for col in 0 ..< width {
                let offset = Int(row * width + col)

                let red = Float(pixels[offset].red)
                let green = Float(pixels[offset].green)
                let blue = Float(pixels[offset].blue)
                let alpha = pixels[offset].alpha
                let luminance = UInt8(0.2126 * red + 0.7152 * green + 0.0722 * blue)
                pixels[offset] = Pixel(red: luminance, green: luminance, blue: luminance, alpha: alpha)
            }
        }

        // return the image

        let outputImage = context.makeImage()!
        completion(UIImage(cgImage: outputImage, scale: image.scale, orientation: image.imageOrientation))
    }
}

struct Pixel: Equatable {
    private var rgba: UInt32

    var red: UInt8 {
        return UInt8((rgba >> 24) & 255)
    }

    var green: UInt8 {
        return UInt8((rgba >> 16) & 255)
    }

    var blue: UInt8 {
        return UInt8((rgba >> 8) & 255)
    }

    var alpha: UInt8 {
        return UInt8((rgba >> 0) & 255)
    }

    init(red: UInt8, green: UInt8, blue: UInt8, alpha: UInt8) {
        rgba = (UInt32(red) << 24) | (UInt32(green) << 16) | (UInt32(blue) << 8) | (UInt32(alpha) << 0)
    }

    static let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Little.rawValue

    static func ==(lhs: Pixel, rhs: Pixel) -> Bool {
        return lhs.rgba == rhs.rgba
    }
}

Clearly, if you want to convert it to absolute black and white, then adjust the algorithm accordingly, but this illustrates a concurrent image buffer manipulation routine.

---

While the above is reasonably fast (again, in optimized release builds), using vImage is even faster. The following is adapted from Converting Color Images to Grayscale:

func grayscale(of image: UIImage) -> UIImage? {
    guard var source = sourceBuffer(for: image) else { return nil }

    defer { free(source.data) }

    var destination = destinationBuffer(for: source)

    // Declare the three coefficients that model the eye's sensitivity
    // to color.
    let redCoefficient: Float = 0.2126
    let greenCoefficient: Float = 0.7152
    let blueCoefficient: Float = 0.0722

    // Create a 1D matrix containing the three luma coefficients that
    // specify the color-to-grayscale conversion.
    let divisor: Int32 = 0x1000
    let fDivisor = Float(divisor)

    var coefficients = [
        Int16(redCoefficient * fDivisor),
        Int16(greenCoefficient * fDivisor),
        Int16(blueCoefficient * fDivisor)
    ]

    // Use the matrix of coefficients to compute the scalar luminance by
    // returning the dot product of each RGB pixel and the coefficients
    // matrix.
    let preBias: [Int16] = [0, 0, 0, 0]
    let postBias: Int32 = 0

    let result = vImageMatrixMultiply_ARGB8888ToPlanar8(
        &source,
        &destination,
        &coefficients,
        divisor,
        preBias,
        postBias,
        vImage_Flags(kvImageNoFlags))

    guard result == kvImageNoError else { return nil }

    defer { free(destination.data) }

    // Create a 1-channel, 8-bit grayscale format that's used to
    // generate a displayable image.
    var monoFormat = vImage_CGImageFormat(
        bitsPerComponent: 8,
        bitsPerPixel: 8,
        colorSpace: Unmanaged.passRetained(CGColorSpaceCreateDeviceGray()),
        bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue),
        version: 0,
        decode: nil,
        renderingIntent: .defaultIntent)

    // Create a Core Graphics image from the grayscale destination buffer.
    let cgImage = vImageCreateCGImageFromBuffer(&destination,
                                                &monoFormat,
                                                nil,
                                                nil,
                                               vImage_Flags(kvImageNoFlags),
                                               nil)?.takeRetainedValue()
    return cgImage.map { UIImage(cgImage: $0) }
}


func sourceBuffer(for image: UIImage) -> vImage_Buffer? {
    guard let cgImage = image.cgImage else { return nil }

    let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).union(.byteOrder32Big)

    var format = vImage_CGImageFormat(bitsPerComponent: 8,
                                      bitsPerPixel: 32,
                                      colorSpace: Unmanaged.passRetained(CGColorSpaceCreateDeviceRGB()),
                                      bitmapInfo: bitmapInfo,
                                      version: 0,
                                      decode: nil,
                                      renderingIntent: .defaultIntent)

    var sourceImageBuffer = vImage_Buffer()
    vImageBuffer_InitWithCGImage(&sourceImageBuffer,
                                 &format,
                                 nil,
                                 cgImage,
                                 vImage_Flags(kvImageNoFlags))

    return sourceImageBuffer

func destinationBuffer(for sourceBuffer: vImage_Buffer) -> vImage_Buffer {
    var destinationBuffer = vImage_Buffer()

    vImageBuffer_Init(&destinationBuffer,
                      sourceBuffer.height,
                      sourceBuffer.width,
                      8,
                      vImage_Flags(kvImageNoFlags))

    return destinationBuffer
}

Answer 2

The vImage conversion to 1 bit is vImageConvert_Planar8ToPlanar1. I recommend using one of the dithered options. You will need to convert your RGB image to grayscale first. In principle, this is vImageMatrixMultiply_ARGB8888ToPlanar8(), though really it probably should involve some more sophisticated colorspace conversion rather than a simple matrix.

If all this sounds too complicated, just use vImageConvert_AnyToAny and it should do the right thing.