How to perform dynamic conversion between mouse coordinates and world coordinates with QtQuick3D?

Question

Description/ Code

I have a Qt Quick 3D View and corresponding scene that was designed to be compiled on Qt 6.3.0

---

import QtQuick
import QtQml
import QtQuick3D
import QtQuick3D.Helpers

Window {
    width: 800
    height: 600
    visible: true
    property var selectedItem
    property bool mousePressed: false
    function multiply_vectors(vec1, vec2) {
        return Qt.vector3d(vec1.x * vec2.x, vec1.y * vec2.y, vec1.z * vec2.z);
        
        
    }
    View3D {
        
        renderMode: View3D.Inline
        camera: camera
        anchors.fill: parent
        width: 800
        height: 600
        x: 0
        y: 0
        id: view
        environment: SceneEnvironment {
            clearColor: "black"
            backgroundMode: SceneEnvironment.Color
            depthTestEnabled: false
            depthPrePassEnabled: true
            
        }
        
        
        Model {
            id: rootEntity
            pickable: true
            source: "#Cube"
            materials: PrincipledMaterial {
                baseColor: "red"
                roughness: 0.1
            }
            position: Qt.vector3d(25.0, 15.0, -60.0)
            scale: Qt.vector3d(1.0, 1.0, 1.0)
        }
        
        PerspectiveCamera {
            id: camera
            position.z: 330.0
            position.y: 0.75
            eulerRotation.x: -12
            
            clipNear: 0.0
            clipFar: 1600.0
        }
        
        
        
        MouseArea {
            acceptedButtons: Qt.LeftButton | Qt.RightButton
            anchors.fill: parent
            id: mouseArea
            
            onPressed: function (mouse) {
                
                var result = view.pick(mouse.x, mouse.y);
                
                if (result.objectHit) {
                    selectedItem = result.objectHit;
                    mousePressed = true;
                } else {
                    mousePressed = false;
                }
            }
            
            onMouseXChanged: function(mouse) {
                if (mousePressed) {
                    var viewCoords = view.mapFromGlobal(mouseArea.mapToGlobal(mouse.x, mouse.y));
                    var sceneCoords = Qt.vector3d(viewCoords.x, viewCoords.y, 0);
                    var worldCoords = view.mapTo3DScene(sceneCoords);
                    worldCoords.z = selectedItem.z
                    selectedItem.position = multiply_vectors(worldCoords, Qt.vector3d(Math.abs(camera.z - selectedItem.z), Math.abs(camera.z - selectedItem.z), 1.0))
                }
            }
            
            onReleased: function (mouse) {
                mousePressed = false
            }
            
        }
        
        Component.onCompleted: {
            camera.lookAt(rootEntity)
        }
    }
}

---

Overview

The use case is that whenever the mouse is pressed while pointing at the cube, whenever the mouse moves it will cause the cube to move along with it to the corresponding point in the 3d Scene.

This works great when looking from a point that is on the same z-axis. However when looking at the object from a point say along the x-axis, the model will move along the x-axis instead of following the mouse position.

Question

How can I modify the business logic in onMouseXChanged: function(mouse) { to correctly transform the matrix (or equivalent transform) to consistently match the mouse position irregardless of the camera's position relative to the Model?

Answer 1

After spending a while experimenting with different approaches, I found that mapping the mouse coordinates to the 3d space wasn't fully supported by the Qt API in terms of when the mouse is not fixed over an active object.

So, instead, the way that I made a workout was by casting a new RayCast each time the mouse moves and storing the offset when the mouse is pressed originally and then translating the item based on the result of the raycast and lining up the offset by translating by the normalized matrix with a small scalar.

onMouseXChanged: function (mouse) {
     if (mousePressed) {
                    if (selectedItem != null) {
                        var result = view.pick(mouse.x, mouse.y)
                        if (result.objectHit) {
                            if (result.objectHit == selectedItem) {
                                var mouseGlobalPos = mouseArea.mapToGlobal(
                                            mouse.x, mouse.y)
                                
                                var mouseViewPos = view.mapFromGlobal(
                                            mouseGlobalPos)

                                var mouseScenePos = result.scenePosition
                                var resultPos = result.position

                                /* here we subtract the result of the new raycast by the starting offset and then normalize 
                                 * the result and multiply it by a scalar 3 to determine the amount of offset the Model 
                                 * under the mouse is from where the mouse was originally pressed, so we can translate it */

                                var differencePos = resultPos.minus(
                                            startMousePressSelectedItemLocalDragOffset).normalized(
                                            ).times(3)
                                selectedItem.position = selectedItem.position.plus(
                                            differencePos)

Answer 2

If I understood you correctly, you need to move the object with the mouse parallel to the camera regardless of the camera position and model scaling? I admit that I don't have a solution, but still it's better than the original code. First of all, do not set the clipNear to 0, it would make the frustum degenerate and break the projection math.

Secondly, I would suppose that the code which sets the object position should look like

selectedItem.position = view.mapTo3DScene(
                        Qt.vector3d(mouse.x, mouse.y,
                                view.mapFrom3DScene(selectedItem.position).z))

The docs say that mapFrom3DScene/mapTo3DScene should interpret the z coordinate as the distance from the near clip plane of the frustum to the mapped position. However when I move it towards the sides of the window the object gets larger, whereas it should get smaller.

Here's the complete code with a few corrections of mine:

import QtQuick
import QtQml
import QtQuick3D
import QtQuick3D.Helpers

Window {
    width: 800
    height: 600
    visible: true
    property var selectedItem
    property bool mousePressed: false

    View3D {
        renderMode: View3D.Inline
        camera: camera
        anchors.fill: parent
        width: 800
        height: 600
        x: 0
        y: 0
        id: view
        environment: SceneEnvironment {
            clearColor: "black"
            backgroundMode: SceneEnvironment.Color
            depthTestEnabled: false
            depthPrePassEnabled: true

        }


        Model {
            id: rootEntity
            pickable: true
            source: "#Cube"
            materials: PrincipledMaterial {
                baseColor: "red"
                roughness: 0.1
            }
            position: Qt.vector3d(25.0, 15.0, -60.0)
            scale: Qt.vector3d(2.0, 1.0, 0.5)
        }

        PerspectiveCamera {
            id: camera
            position.z: 330.0
            position.y: 100
            position.x: 700
            eulerRotation.x: -12

            // Note 1: clipNear shouldn't be 0, otherwise
            // it would break the math inside the projection matrix
            clipNear: 1.0
            clipFar: 1600.0
        }



        MouseArea {
            acceptedButtons: Qt.LeftButton | Qt.RightButton
            anchors.fill: parent
            id: mouseArea

            onPressed: function (mouse) {

                var result = view.pick(mouse.x, mouse.y);

                if (result.objectHit) {
                    selectedItem = result.objectHit;
                    mousePressed = true;
                } else {
                    mousePressed = false;
                }
            }

            onPositionChanged: function(mouse) {
                if (mousePressed) {
                    // Note 2: recalculate the position, since MouseArea has
                    // the same geometry as View3D we can use coords directly
                    selectedItem.position = view.mapTo3DScene(
                        Qt.vector3d(mouse.x, mouse.y,
                                view.mapFrom3DScene(selectedItem.position).z))
                }
            }

            onReleased: function (mouse) {
                mousePressed = false
            }

        }

        Component.onCompleted: {
            camera.lookAt(rootEntity)
        }
    }
}