Make VoiceOver ignore a range within AttributedString

Question

I have an inline hyperlink component that's written in SwiftUI that uses an AttributedString internally to apply styles to certain parts of the text as well as adding an arrow icon next to the links optionally. I would like the VoiceOver to ignore the arrow character completely instead of announcing it as "north east arrow". I've tried most of the accessibility related properties of AttributedString but none seemed to work and overriding accessibilityLabel on the Text element seems to break the VoiceOver announcement for links.

Here is the related code:

    private var attributedString: AttributedString {
        var attributedString = AttributedString(text)

        let fontProvider = FontProvider()
        let colorProvider = ColorProvider()

        attributedString.font = fontProvider.font(for: .bodyLarge)
        attributedString.foregroundColor = colorProvider.black

        for (substring, url) in links {
            guard let range = attributedString.range(of: substring) else {
                assertionFailure("The substring '\(substring)' was not found in the provided text.")
                break
            }

            attributedString[range].link = url
            attributedString[range].font = fontProvider.font(for: .linkLarge)
            attributedString[range].foregroundColor = colorProvider.primary
            attributedString[range].underlineStyle = .single

            if isExternal {
                var arrowString = AttributedString(" \u{2197}")
                arrowString.font = Font.system(.headline, design: .monospaced)
                arrowString.foregroundColor = colorProvider.primary
                attributedString.insert(arrowString, at: range.upperBound)
            }
        }

        return attributedString
    }