Three.js: How to calculate the minimum bounding rectangle of an object in a camera image?

Question

I'd like to calculate and draw the minimum bounding rectangle (MBR) of an object in the image of a camera (2D projection). The following picture shows the MBR of the cube object of the example provided below. I took a screenshot of the camera image and drew the MBR (red rectangle) manually by eye. How can I calculate the MBR (size and position in the camera image) programmatically and draw it in the canvas of the camera image (in real time)?

Edit: I added a MBR to my example based on this related solution, but I drew the MBR on a 2D overlay canvas. It works fine till you move the camera (using the mouse). Then the MBR is not correct (shifted and wrong size). Does the calculation not fit to my example or is there a bug in my implementation?

Note: I do not only want to draw the MBR but also find out the 2D coordinates and size on the camera image.

function computeScreenSpaceBoundingBox(mesh, camera) {
  var vertices = mesh.geometry.vertices;
  var vertex = new THREE.Vector3();
  var min = new THREE.Vector3(1, 1, 1);
  var max = new THREE.Vector3(-1, -1, -1);

  for (var i = 0; i < vertices.length; i++) {
    var vertexWorldCoord = vertex.copy(vertices[i]).applyMatrix4(mesh.matrixWorld);
    var vertexScreenSpace = vertexWorldCoord.project(camera);
    min.min(vertexScreenSpace);
    max.max(vertexScreenSpace);
  }

  return new THREE.Box2(min, max);
}

function normalizedToPixels(coord, renderWidthPixels, renderHeightPixels) {
  var halfScreen = new THREE.Vector2(renderWidthPixels/2, renderHeightPixels/2)
  return coord.clone().multiply(halfScreen);
}

const scene = new THREE.Scene();
const light = new THREE.DirectionalLight( 0xffffff, 1 );
light.position.set( 1, 1, 0.5 ).normalize();
scene.add( light );
scene.add(new THREE.AmbientLight(0x505050));

const geometry = new THREE.BoxGeometry( 1, 1, 1 );
const material = new THREE.MeshLambertMaterial( { color: 0x00ff00 } );
const cube = new THREE.Mesh( geometry, material );
scene.add( cube );

const renderWidth = window.innerWidth;
const renderHeight = window.innerHeight;
const camera = new THREE.PerspectiveCamera( 75, renderWidth / renderHeight, 0.1, 1000 );
camera.position.z = 10;

const controls = new THREE.OrbitControls( camera );
controls.update();

const renderer = new THREE.WebGLRenderer();
renderer.setSize( window.innerWidth, window.innerHeight );
document.body.appendChild( renderer.domElement );

const clock = new THREE.Clock();

const overlayCanvas = document.getElementById('overlay');
overlayCanvas.width = renderWidth;
overlayCanvas.height = renderHeight;
const overlayCtx = overlayCanvas.getContext('2d');
overlayCtx.lineWidth = 4;
overlayCtx.strokeStyle = 'red';


const animate = function () {
  const time = clock.getElapsedTime() * 0.5;
  
  requestAnimationFrame( animate );

  cube.rotation.x += 0.01;
  cube.rotation.y += 0.01;
  cube.position.x = Math.sin(time) * 5

  controls.update();
  renderer.render( scene, camera );
  
  const boundingBox2D = computeScreenSpaceBoundingBox(cube, camera);
  // Convert normalized screen coordinates [-1, 1] to pixel coordinates:
  const {x: w, y: h} = normalizedToPixels(boundingBox2D.getSize(), renderWidth, renderHeight); 
  const {x, y} =
    normalizedToPixels(boundingBox2D.min, renderWidth, renderHeight)
      .add(new THREE.Vector2(renderWidth / 2, renderHeight / 2)); 
  
  overlayCtx.clearRect(0, 0, renderWidth, renderHeight);
  overlayCtx.strokeRect(x, y, w, h);
};

animate();

body {
  margin: 0px;
  background-color: #000000;
  overflow: hidden;
}

#overlay {
  position: absolute;
}

Note: Please write a comment if my question is unclear so that I know what to describe in more detail.

Three.js: How to calculate the minimum bounding rectangle of an object in a camera image?

Answers (1)

Related Questions