Rendering MTLTexture on MTKView is not keeping aspect ratio

Question

I have a texture that's 1080x1920 pixels. And I'm trying to render it on a MTKView that isn't the same aspect ratio. (i.e iPad/iPhone X full screen).

This is how I'm rendering the texture for the MTKView:

private func render(_ texture: MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) {
    guard let currentRenderPassDescriptor = metalView?.currentRenderPassDescriptor,
            let currentDrawable = metalView?.currentDrawable,
            let renderPipelineState = renderPipelineState,
            let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) else {
                semaphore.signal()
                return
        }

    encoder.pushDebugGroup("RenderFrame")
    encoder.setRenderPipelineState(renderPipelineState)
    encoder.setFragmentTexture(texture, index: 0)
    encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1)
    encoder.popDebugGroup()
    encoder.endEncoding()

    // Called after the command buffer is scheduled
    commandBuffer.addScheduledHandler { [weak self] _ in
        guard let strongSelf = self else {
            return
        }
        strongSelf.didRender(texture: texture)
        strongSelf.semaphore.signal()
    }

    commandBuffer.present(currentDrawable)
    commandBuffer.commit()
}

I want the texture to be rendered like .scaleAspectFill on a UIView and I'm trying to learn Metal so I'm not sure where I should be looking for this (the .metal file, the pipeline, the view itself, the encoder, etc.)

Thanks!

Edit: Here is the shader code:

#include <metal_stdlib> using namespace metal;

typedef struct {
    float4 renderedCoordinate [[position]];
    float2 textureCoordinate; } TextureMappingVertex;

vertex TextureMappingVertex mapTexture(unsigned int vertex_id [[ vertex_id ]]) {
    float4x4 renderedCoordinates = float4x4(float4( -1.0, -1.0, 0.0, 1.0 ),
                                            float4(  1.0, -1.0, 0.0, 1.0 ),
                                            float4( -1.0,  1.0, 0.0, 1.0 ),
                                            float4(  1.0,  1.0, 0.0, 1.0 ));

    float4x2 textureCoordinates = float4x2(float2( 0.0, 1.0 ),
                                           float2( 1.0, 1.0 ),
                                           float2( 0.0, 0.0 ),
                                           float2( 1.0, 0.0 ));
    TextureMappingVertex outVertex;
    outVertex.renderedCoordinate = renderedCoordinates[vertex_id];
    outVertex.textureCoordinate = textureCoordinates[vertex_id];

    return outVertex; }

fragment half4 displayTexture(TextureMappingVertex mappingVertex [[ stage_in ]],texture2d<float, access::sample> texture [[ texture(0) ]]) {
    constexpr sampler s(address::clamp_to_edge, filter::linear);

    return half4(texture.sample(s, mappingVertex.textureCoordinate));
}

Answer 1

On the other hand, I decided to do texture coordinates calculation in renderer using CPU and inject the matrix into shader functions rather than doing the same calculation for each pixel. So this is how the code looks like now.

Shader

#include <metal_stdlib>
using namespace metal;
#include <BridgingHeader.h>

struct VertexOut {
    float4 position [[position]];
    float2 texture;
};

vertex VertexOut textureWithCustomCoordinatesVertex(const device Vertex *vertexArray [[buffer(0)]],
                                                    unsigned int vid [[vertex_id]],
                                                    constant float4x2 &textureCoordinates [[ buffer(1) ]]
                                                    ) {
    VertexOut output;
    output.position = float4(vertexArray[vid].position, 0, 1);
    output.texture = textureCoordinates[vid];

    return output;
}

fragment half4 textureWithCustomCoordinatesFragment(VertexOut input [[stage_in]],
                                                    texture2d<float> texture [[ texture(0) ]],
                                                    sampler sampler2d [[ sampler(0) ]]
                                                    ) {
    float4 color = texture.sample(sampler2d, input.texture);
    return half4(color.r, color.g, color.b, 1);
}

Renderer

final class MyRenderer: NSObject, MTKViewDelegate {
    // ... 

    private func getTextureAspectFillCoordinates(
        inputTexture: MTLTexture,
        drawableTexture: MTLTexture
    ) -> simd_float4x2 {
        // 1. Get ratios for both textures
        let textureWidth: simd_float1 = .init(inputTexture.width)
        let textureHeight: simd_float1 = .init(inputTexture.height)
        let textureAspectRatio: simd_float1 = textureWidth / textureHeight
        
        let drawableWidth: simd_float1 = .init(drawableTexture.width)
        let drawableHeight: simd_float1 = .init(drawableTexture.height)
        let drawableAspectRatio: simd_float1 = drawableWidth / drawableHeight
        
        // 2. Declare output points
        let topLeft: simd_float2
        let topRight: simd_float2
        let bottomLeft: simd_float2
        let bottomRight: simd_float2
        
        // 3. Check if texture's ratio is greater than drawable's
        if textureAspectRatio > drawableAspectRatio {
            // We need to draw whole height and clip width
            let left: simd_float1 = abs(textureWidth - drawableWidth) / 2.0
            let right = textureWidth - left
            
            let normalizedLeft = left / textureWidth
            let normalizedRight = right / textureWidth
            
            topLeft = .init(normalizedLeft, 0)
            topRight = .init(normalizedRight, 0)
            bottomLeft = .init(normalizedLeft, 1)
            bottomRight = .init(normalizedRight, 1)
        } else {
            // We need to draw whole width and clip height
            let top: simd_float1 = abs(textureHeight - drawableHeight) / 2.0
            let bottom = textureHeight - top
            
            let normalizedTop = top / textureHeight
            let normalizedBottom = bottom / textureHeight
        
            topLeft = .init(0, normalizedTop)
            topRight = .init(1, normalizedTop)
            bottomLeft = .init(0, normalizedBottom)
            bottomRight = .init(1, normalizedBottom)
        }
        
        return .init(columns: (topLeft, bottomLeft, bottomRight, topRight))
    }

    // ...

    func draw(in view: MTKView) {
        guard let pixelBuffer else { return }
        guard let textureCache,
              let commandBuffer = context.commandQueue.makeCommandBuffer(),
              let renderPassDescriptor = view.currentRenderPassDescriptor
        else {
            assertionFailure("Can't perform draw")
            return
        }
        
        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 1.0)
        renderPassDescriptor.colorAttachments[0].loadAction = .clear
        renderPassDescriptor.colorAttachments[0].storeAction = .store
        guard let encoder = commandBuffer.makeRenderCommandEncoder(
            descriptor: renderPassDescriptor
        ) else {
            assertionFailure("Could not create encoder")
            return
        }
        guard let cvTexture = getCVTexture(from: pixelBuffer, and: textureCache),
              let inputTexture = CVMetalTextureGetTexture(cvTexture)
        else {
            assertionFailure("Failed to create metal textures")
            return
        }
        
        encoder.setRenderPipelineState(pipelineState)
        autoreleasepool {
            guard let drawable = view.currentDrawable else { return }
            
            var textureCoordinates = getTextureAspectFillCoordinates(
                inputTexture: inputTexture,
                drawableTexture: drawable.texture
            )
            
            encoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0)
            encoder.setVertexBytes(
                &textureCoordinates,
                length: MemoryLayout<simd_float4x2>.size,
                index: 1
            )
            encoder.setFragmentTexture(inputTexture, index: 0)
            encoder.setFragmentSamplerState(samplerState, index: 0)
            
            encoder.drawIndexedPrimitives(
                type: .triangle,
                indexCount: indices.count,
                indexType: .uint16,
                indexBuffer: indexBuffer,
                indexBufferOffset: 0
            )
            encoder.endEncoding()
            
            commandBuffer.present(drawable)
            commandBuffer.commit()
        }
    }
}

Answer 2

1awuesterose's answer is working. The only thing I'd add is to use abs(tex.width - r_tex.width).

This is how my code looks if it helps.

#ifndef BridgingHeader_h
#define BridgingHeader_h
#include <simd/simd.h>

struct Vertex {
    vector_float2 position;
};

#endif /* BridgingHeader_h */

#include <metal_stdlib>
using namespace metal;
#include <BridgingHeader.h>

struct VertexOut {
    float4 position [[position]];
    float2 texture;
};

vertex VertexOut textureVertexAspectFill(const device Vertex *vertexArray [[buffer(0)]],
                                         unsigned int vid [[vertex_id]],
                                         texture2d<float> texture [[ texture(0) ]],
                                         texture2d<float> resolveTexture [[ texture(1) ]]) {
    float textureWidth = texture.get_width();
    float textureHeight = texture.get_height();
    
    float textureAspectRatio = textureWidth / textureHeight;
    
    float resolveWidth = resolveTexture.get_width();
    float resolveHeight = resolveTexture.get_height();
    
    float resolveAspectRatio = resolveWidth / resolveHeight;
    
    bool isTextureRatioGreater = textureAspectRatio > resolveAspectRatio;
    
    float2 topLeft;
    float2 topRight;
    float2 bottomLeft;
    float2 bottomRight;
    
    if (isTextureRatioGreater) {
        float left = abs(textureWidth - resolveWidth) / 2.0;
        float right = textureWidth - left;
        
        float normalizedLeft = left / textureWidth;
        float normalizedRight = right / textureWidth;
        
        topLeft = float2(normalizedLeft, 0);
        topRight = float2(normalizedRight, 0);
        bottomLeft = float2(normalizedLeft, 1);
        bottomRight = float2(normalizedRight, 1);
    } else {
        float top = abs(textureHeight - resolveHeight) / 2.0;
        float bottom = textureHeight - top;
        
        float normalizedTop = top / textureHeight;
        float normalizedBottom = bottom / textureHeight;
        
        topLeft = float2(0, normalizedTop);
        topRight = float2(1, normalizedTop);
        bottomLeft = float2(0, normalizedBottom);
        bottomRight = float2(1, normalizedBottom);
    }
    
    float4x2 textureCoordinates = float4x2(topLeft,
                                           bottomLeft,
                                           bottomRight,
                                           topRight);

    VertexOut output;
    output.position = float4(vertexArray[vid].position, 0, 1);
    output.texture = textureCoordinates[vid];

    return output;
}

fragment half4 textureFragmentAspectFill(VertexOut input [[stage_in]],
                               texture2d<float> texture [[ texture(0) ]]) {
    constexpr sampler defaultSampler;
    float4 color = texture.sample(defaultSampler, input.texture);
    return half4(color.r, color.g, color.b, 1);
}

    // ...

    private let vertexBuffer: any MTLBuffer
    private let indexBuffer: any MTLBuffer
    var indices: [UInt16] = [
        0, 1, 2,
        2, 3, 0
    ]

    // ...

    let vertices = [
        Vertex(position: [-1, 1]), // Top left
        Vertex(position: [-1, -1]), // Bottom left
        Vertex(position: [1, -1]), // Bottom right
        Vertex(position: [1, 1]) // Top right
    ]
        
    vertexBuffer = context.device.makeBuffer(
        bytes: vertices,
        length: vertices.count * MemoryLayout<Vertex>.stride,
        options: []
    )!
        
    indexBuffer = context.device.makeBuffer(
        bytes: indices,
        length: indices.count * MemoryLayout<UInt16>.size
    )!

    // ...

    public func draw(in view: MTKView) {
        guard let pixelBuffer else { return }
        guard let textureCache,
              let commandBuffer = context.commandQueue.makeCommandBuffer(),
              let renderPassDescriptor = view.currentRenderPassDescriptor
        else {
            assertionFailure("Can't perform draw")
            return
        }
        
        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 1.0)
        renderPassDescriptor.colorAttachments[0].loadAction = .clear
        renderPassDescriptor.colorAttachments[0].storeAction = .store
        guard let encoder = commandBuffer.makeRenderCommandEncoder(
            descriptor: renderPassDescriptor
        ) else {
            assertionFailure("Could not create encoder")
            return
        }
        guard let cvTexture = getCVTexture(from: pixelBuffer, and: textureCache),
              let inputTexture = CVMetalTextureGetTexture(cvTexture)
        else {
            assertionFailure("Failed to create metal textures")
            return
        }
        
        encoder.setRenderPipelineState(pipelineState)
        autoreleasepool {
            guard let drawable = view.currentDrawable else { return }
            
            encoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0)
            encoder.setVertexTexture(inputTexture, index: 0)
            encoder.setVertexTexture(drawable.texture, index: 1)
            encoder.setFragmentTexture(inputTexture, index: 0)
            
            encoder.drawIndexedPrimitives(
                type: .triangle,
                indexCount: indices.count,
                indexType: .uint16,
                indexBuffer: indexBuffer,
                indexBufferOffset: 0
            )
            encoder.endEncoding()
            
            commandBuffer.present(drawable)
            commandBuffer.commit()
        }
    }

Answer 3

A few general things to start with when dealing with Metal textures or Metal in general:

• You should take into account the difference between points and pixels, refer to the documentation here. The frame property of a UIView subclass (as MTKView is one) always gives you the width and the height of the view in points.

• The mapping from points to actual pixels is controlled through the contentScaleFactor option. The MTKView automatically selects a texture with a fitting aspect ratio that matches the actual pixels of your device. For example, the underlying texture of a MTKView on the iPhone X would have a resolution of 2436 x 1125 (the actual display size in pixels). This is documented here: "The MTKView class automatically supports native screen scale. By default, the size of the view’s current drawable is always guaranteed to match the size of the view itself."

• As documented here, the .scaleAspectFill option "scale[s] the content to fill the size of the view. Some portion of the content may be clipped to fill the view’s bounds". You want to simulate this behavior.

• Rendering with Metal is nothing more than "drawing" to the resolve texture, which is automatically set by the MTKView. However, you still have full control and could do it on your own by manually creating textures and setting them in your renderPassDescriptor. But you don't need to care about this right now. The single thing you should care about is what, where and which part of the 1080x1920 pixels texture in your resolve texture you want to render in your resolve texture (which might have a different aspect ratio). We want to fully fill ("scaleAspectFill") the resolve texture, so we leave the renderedCoordinates in your fragment shader as they are. The are defining a rectangle over the whole resolve texture, which means the fragment shader is called for every single pixel in the resolve texture. Following, we will simply change the texture coordinates.

• Let's define the aspect ratio as ratio = width / height, the resolve texture as r_tex and the texture you want to render as tex.

• So assuming your resolve texture does not have the same aspect ratio, there are two possible scenarios:

  1. The aspect ratio of your texture that you want to render is larger than the aspect ratio of your resolve texture (the texture Metal renders to), that means the texture you want to render has a larger width than the resolve texture. In this case we leave the y values of the coordinate as they are. The x values of texture coordinates will be changed:

     x_left  = 0 + ((tex.width - r_tex.width) / 2.0)
     x_right = tex_width - ((tex.width - r_tex_width) / 2.0)
     

     These values must be normalized because the texture samples needs coordinates in the range from 0 to 1:

     x_left  = x_left / tex.width
     x_right = x_right / tex.width
     

     We have our new texture coordinates:

     topLeft = float2(x_left,0)
     topRight = float2(x_right,0)
     bottomLeft = float2(x_left,1)
     bottomRight = float2(x_right,1)
     

     This will have the effect that nothing of the top or the bottom of your texture will be cut off, but some outer parts at the left and right side will be clipped, i.e. not visible.

  2. The aspect ratio of your texture that you want to render is smaller than the aspect ratio of your resolve texture. The procedure is the same as with first scenario, but this time we will change the y coordinates

This should render your texture so that the resolve texture is completely filled and the aspect ratio of your texture is maintained on the x-axis. Maintaining the y-axis will work similarly. Additionally you have to check which side of the texture is larger/smaller and incorporate this in your calculation. This will clip parts of your texture as it would be when using scaleAspectFill. Be aware that the above solution is untested. But I hope it is helpful. Be sure to visit Metal Best Practices documentation from time to time, it's very helpful to get the basic concepts right. Have fun with Metal!

Answer 4

So your vertex shader pretty directly dictates that the source texture be stretched to the dimensions of the viewport. You are rendering a quad that fills the viewport, because its coordinates are at the extremes ([-1, 1]) of the Normalized Device Coordinate system in the horizontal and vertical directions.

And you are mapping the source texture corner-to-corner over that same range. That's because you specify the extremes of texture coordinate space ([0, 1]) for the texture coordinates.

There are various approaches to achieve what you want. You could pass the vertex coordinates in to the shader via a buffer, instead of hard-coding them. That way, you can compute the appropriate values in app code. You'd compute the desired destination coordinates in the render target, expressed in NDC. So, conceptually, something like left_ndc = (left_pixel / target_width) * 2 - 1, etc.

Alternatively, and probably easier, you can leave the shader as-is and change the viewport for the draw operation to target only the appropriate portion of the render target.