CMake MacOS universal binary build: Link to a certain lib only for x86_64 and not for ARM

Question

I'm setting up a MacOS universal binary build (x86_64 & arm64) for a JUCE based audio plugin project. The x86_64 build has some IPP libraries (which are x86 only) as a dependency while the ARM build uses some ARM specific replacement routines written by us. So I need to set up CMake to build the plugin as universal binary but only link to IPP libs for the x86_64 part. My approach so far looks like that

# somewhere at the top of my CMakeLists.txt
set(CMAKE_OSX_ARCHITECTURES arm64 x86_64)
set(ONLY_ACTIVE_ARCH        NO)

# somewhere later the IPP stuff is handled
add_library(ALL_IPP INTERFACE)

set(IPP_ROOT "/opt/intel/ipp")

target_include_directories(ALL_IPP INTERFACE "${IPP_ROOT}/include")

add_library(ippi    STATIC IMPORTED GLOBAL)
add_library(ipps    STATIC IMPORTED GLOBAL)
add_library(ippvm   STATIC IMPORTED GLOBAL)
add_library(ippcore STATIC IMPORTED GLOBAL)

set (IPP_LIB "${IPP_ROOT}/lib")

set_target_properties(ippi    PROPERTIES IMPORTED_LOCATION ${IPP_LIB}/libippi.a    OSX_ARCHITECTURES x86_64)
set_target_properties(ipps    PROPERTIES IMPORTED_LOCATION ${IPP_LIB}/libipps.a    OSX_ARCHITECTURES x86_64)
set_target_properties(ippvm   PROPERTIES IMPORTED_LOCATION ${IPP_LIB}/libippvm.a   OSX_ARCHITECTURES x86_64)
set_target_properties(ippcore PROPERTIES IMPORTED_LOCATION ${IPP_LIB}/libippcore.a OSX_ARCHITECTURES x86_64)

target_link_libraries(ALL_IPP INTERFACE ippi ipps ippvm ippcore)

# somewhere later
target_link_libraries(myPlugin PRIVATE ALL_IPP)

This approach leads to build warnings like

ld: warning: ignoring file /opt/intel/ipp/lib/libippi.a, building for macOS-arm64 but attempting to link with file built for unknown-x86_64

I'm not that experienced with CMake, so its totally possible that I'm choosing a completely wrong approach here or that there is just a little stupid error. All help is greatly appreciated!