How to test if a point is in a rectangle area which rotates an angle?

Question

I am trying to test if a point is inside a rectangle area that rotates an angle around (x, y), like the image below. This is language agnostic problem but I am working with HTML5 canvas now.

Suppose the point we need to test is (x1, y1), the width of the rectangle is 100 and the height is 60. In normal cartesian coordinate system the rectangle ABCD top left point A is (canvas.width / 2, canvas.height / 2 -rect.height/2). I assume that (canvas.width / 2, canvas.height / 2) is at the middle of line AB where B is (canvas.width / 2, canvas.height / 2 + rect.height /2).

I have read some resources here and wrote a test project, but it doesn't test the correct area. In my test project I want the this effect:

if the mouse is on a point that is within the range of the testing rectangle area a dot will be displayed around the mouse. If it is outside the rectangle nothing will be displayed.

However my test project looks like this: (Note that although I used the vector based technique to test the point in a rotated rectangle area, the test area remains the rectangle before rotation)

// Detecting a point is in a rotated rectangle area
// using vector based method
const canvas = document.getElementById('canvas');
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
const ctx = canvas.getContext('2d');

class Rectangle {
	constructor(x, y, width, height) {
  	this.x = x;
    this.y = y;
    this.width = width;
    this.height = height;
    this.searchPoint = { x: 0, y: 0};
    this.binding();
  }
  
  binding() {
  	let self = this;
    window.addEventListener('mousemove', e => {
      if (!e) return;
      let rect = canvas.getBoundingClientRect();
      let mx = e.clientX - rect.left - canvas.clientLeft;
      let my = e.clientY - rect.top - canvas.clientTop;
      self.searchPoint = { x: mx, y: my };
    });
	}
}

let rect = new Rectangle(canvas.width /2, canvas.height /2 - 30, 100, 60);

function vector(p1, p2) {
    return {
            x: (p2.x - p1.x),
            y: (p2.y - p1.y)
    };
}

function point(x, y) {
	return { x, y };
}

// Vector dot operation
function dot(a, b) {
  return a.x * b.x + a.y * b.y;
}

function pointInRect(p, rect, angle) {
	let a = newPointTurningAngle(0, -rect.height / 2, angle);
	let b = newPointTurningAngle(0, rect.height / 2, angle);
  let c = newPointTurningAngle(rect.width, rect.height / 2, angle);
	let AB = vector(a, b);
  let AM = vector(a, p);
  let BC = vector(b, c);
  let BM = vector(b, p);
  let dotABAM = dot(AB, AM);
  let dotABAB = dot(AB, AB);
  let dotBCBM = dot(BC, BM);
  let dotBCBC = dot(BC, BC);
  
  return 0 <= dotABAM && dotABAM <= dotABAB && 0 <= dotBCBM && dotBCBM <= dotBCBC;
}

function drawLine(x, y) {
	ctx.strokeStyle = 'black';
	ctx.lineTo(x, y);
  ctx.stroke();
}

function text(text, x, y) {
	ctx.font = "18px serif";
  ctx.fillText(text, x, y);
}

function newPointTurningAngle(nx, ny, angle) {
	return {
  	x: nx * Math.cos(angle) - ny * Math.sin(angle),
    y: nx * Math.sin(angle) + ny * Math.cos(angle)
  };
}

function animate() {
	ctx.clearRect(0, 0, canvas.width, canvas.height);
	ctx.setTransform(1, 0, 0, 1, 0, 0);
	ctx.moveTo(canvas.width / 2, 0);
  drawLine(canvas.width /2, canvas.height / 2);
  
  ctx.moveTo(0, canvas.height / 2);
  drawLine(canvas.width / 2, canvas.height /2);
  
	let angle = -Math.PI / 4;
	ctx.setTransform(Math.cos(angle), Math.sin(angle), -Math.sin(angle), Math.cos(angle), canvas.width / 2, canvas.height / 2);
  //ctx.setTransform(1, 0, 0, 1, canvas.width/2, canvas.height / 2);
	ctx.strokeStyle = 'red';
  ctx.strokeRect(0, -rect.height / 2, rect.width, rect.height);
  
	let p = newPointTurningAngle(rect.searchPoint.x - canvas.width / 2, rect.searchPoint.y - canvas.height / 2, angle);

	let testResult = pointInRect(p, rect, angle);
	if (testResult) {
  	ctx.setTransform(1, 0, 0, 1, 0, 0);
    ctx.beginPath();
  	ctx.fillStyle = 'black';
  	ctx.arc(rect.searchPoint.x, rect.searchPoint.y, 5, 0, Math.PI * 2);
    ctx.fill();
  }
  
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  text('searchPoint x: ' + rect.searchPoint.x + ', y: ' + rect.searchPoint.y, 60, 430);
  text('x: ' + canvas.width / 2 + ', y: ' + canvas.height / 2, 60, 480);
  
  requestAnimationFrame(animate);
}

animate();

<canvas id='canvas'></canvas>

Updated Solution

I am still using the vector based method as followed:

0 <= dot(AB,AM) <= dot(AB,AB) &&
0 <= dot(BC,BM) <= dot(BC,BC)

Now I have changed the point's rotated angle and the corner point coordinates so the point can be detected in the rectangle. The corner points are already in the rotated coordinate system so they don't need to be translated, however the point of the mouse location needs to be translated before testing it in the rectangle area.

In setTransform method the angle rotated is positive when rotated clockwise, the form is :

ctx.setTransform(angle_cosine, angle_sine, -angle_sine, angle_cosine, x, y);

So when calculating the point's new coordinate after rotating an angle, the formula need to change to this so that the angle is also positive when rotated clockwise:

 new_x = x * angle_cosine + y * angle_sine;
 new_y = -x * angle_sine + y * angle_cos;

// Detecting a point is in a rotated rectangle area
// using vector based method
const canvas = document.getElementById('canvas');
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
const ctx = canvas.getContext('2d');

class Rectangle {
	constructor(x, y, width, height) {
  	this.x = x;
    this.y = y;
    this.width = width;
    this.height = height;
    this.searchPoint = { x: 0, y: 0};
    this.binding();
  }
  
  binding() {
  	let self = this;
    window.addEventListener('mousemove', e => {
      if (!e) return;
      let rect = canvas.getBoundingClientRect();
      let mx = e.clientX - rect.left - canvas.clientLeft;
      let my = e.clientY - rect.top - canvas.clientTop;
      self.searchPoint = { x: mx, y: my };
    });
	}
}

let rect = new Rectangle(canvas.width /2, canvas.height /2 - 30, 100, 60);

function vector(p1, p2) {
    return {
            x: (p2.x - p1.x),
            y: (p2.y - p1.y)
    };
}

function point(x, y) {
	return { x, y };
}

// Vector dot operation
function dot(a, b) {
  return a.x * b.x + a.y * b.y;
}

function pointInRect(p, rect) {
  let a = { x: 0, y: -rect.height / 2};
  let b = { x: 0, y: rect.height / 2};
  let c = { x: rect.width, y: rect.height / 2};
  text('P x: ' + p.x.toFixed() + ', y: ' + p.y.toFixed(), 60, 430);
  text('A x: ' + a.x.toFixed() + ', y: ' + a.y.toFixed(), 60, 455);
  text('B x: ' + b.x.toFixed() + ', y: ' + b.y.toFixed(), 60, 480);
	let AB = vector(a, b);
  let AM = vector(a, p);
  let BC = vector(b, c);
  let BM = vector(b, p);
  let dotABAM = dot(AB, AM);
  let dotABAB = dot(AB, AB);
  let dotBCBM = dot(BC, BM);
  let dotBCBC = dot(BC, BC);
  
  return 0 <= dotABAM && dotABAM <= dotABAB && 0 <= dotBCBM && dotBCBM <= dotBCBC;
}

function drawLine(x, y) {
	ctx.strokeStyle = 'black';
	ctx.lineTo(x, y);
  ctx.stroke();
}

function text(text, x, y) {
	ctx.font = "18px serif";
  ctx.fillText(text, x, y);
}

function newPointTurningAngle(nx, ny, angle) {
	let cos = Math.cos(angle);
  let sin = Math.sin(angle);
	return {
  	x: nx * cos + ny * sin,
    y: -nx * sin + ny * cos
  };
}

function animate() {
	ctx.clearRect(0, 0, canvas.width, canvas.height);
	ctx.setTransform(1, 0, 0, 1, 0, 0);
	ctx.moveTo(canvas.width / 2, 0);
  drawLine(canvas.width /2, canvas.height / 2);
  
  ctx.moveTo(0, canvas.height / 2);
  drawLine(canvas.width / 2, canvas.height /2);
  
    let angle = - Math.PI / 4;
	ctx.setTransform(Math.cos(angle), Math.sin(angle), -Math.sin(angle), Math.cos(angle), canvas.width / 2, canvas.height / 2);
	ctx.strokeStyle = 'red';
  ctx.strokeRect(0, -rect.height / 2, rect.width, rect.height);
  
  	let p = newPointTurningAngle(rect.searchPoint.x - canvas.width / 2, rect.searchPoint.y - canvas.height / 2, angle);
    
  ctx.setTransform(1, 0, 0, 1, 0, 0);
	let testResult = pointInRect(p, rect);
  
	if (testResult) {
    ctx.beginPath();
  	ctx.fillStyle = 'black';
  	ctx.arc(rect.searchPoint.x, rect.searchPoint.y, 5, 0, Math.PI * 2);
    ctx.fill();
  }
  
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  text('searchPoint x: ' + rect.searchPoint.x + ', y: ' + rect.searchPoint.y, 60, 412);
  text('x: ' + canvas.width / 2 + ', y: ' + canvas.height / 2, 60, 510);
  
  requestAnimationFrame(animate);
}

animate();

<canvas id='canvas'></canvas>

Answer 1

The browser always report mouse position untransformed (==unrotated).

So to test if the mouse is inside a rotated rectangle, you can:

• Get the unrotated mouse position from the mouse event (relative to the canvas).
• Rotate the mouse x,y versus the rotation point by the same rotation as the rectangle.
• Test if the mouse is inside the rectangle. Now that both the rect and the mouse position have been similarly rotated, you can just test as if the mouse and rect were unrotated.

Annotated code and a Demo:

var canvas=document.getElementById("canvas");
var ctx=canvas.getContext("2d");
var cw=canvas.width;
var ch=canvas.height;
function reOffset(){
    var BB=canvas.getBoundingClientRect();
    offsetX=BB.left;
    offsetY=BB.top;        
}
var offsetX,offsetY;
reOffset();
window.onscroll=function(e){ reOffset(); }
window.onresize=function(e){ reOffset(); }

var isDown=false;
var startX,startY;

var rect=makeRect(50,20,35,20,Math.PI/4,60,30);

function makeRect(x,y,w,h,angle,rotationPointX,rotationPointY){
    return({
        x:x,y:y,width:w,height:h,
        rotation:angle,rotationPoint:{x:rotationPointX,y:rotationPointY},
    });
}

drawRect(rect);

$("#canvas").mousedown(function(e){handleMouseDown(e);});

function drawRect(r){
    var rx=r.rotationPoint.x;
    var ry=r.rotationPoint.y;
    // demo only, draw the rotation point
    dot(rx,ry,'blue');
    // draw the rotated rect
    ctx.translate(rx,ry);
    ctx.rotate(r.rotation);
    ctx.strokeRect(rect.x-rx,rect.y-ry,r.width,r.height);
    // always clean up, undo the transformations (in reverse order)
    ctx.rotate(-r.rotation);
    ctx.translate(-rx,-ry);
}

function dot(x,y,fill){
    ctx.fillStyle=fill;
    ctx.beginPath();
    ctx.arc(x,y,3,0,Math.PI*2);
    ctx.fill();
}

function handleMouseDown(e){
    // tell the browser we're handling this event
    e.preventDefault();
    e.stopPropagation();
    // get mouse position relative to canvas
    mouseX=parseInt(e.clientX-offsetX);
    mouseY=parseInt(e.clientY-offsetY);
    // rotate the mouse position versus the rotationPoint
    var dx=mouseX-rect.rotationPoint.x;
    var dy=mouseY-rect.rotationPoint.y;
    var mouseAngle=Math.atan2(dy,dx);
    var mouseDistance=Math.sqrt(dx*dx+dy*dy);
    var rotatedMouseX=rect.rotationPoint.x+mouseDistance*Math.cos(mouseAngle-rect.rotation);
    var rotatedMouseY=rect.rotationPoint.y+mouseDistance*Math.sin(mouseAngle-rect.rotation);
    // test if rotated mouse is inside rotated rect
    var mouseIsInside=rotatedMouseX>rect.x &&
        rotatedMouseX<rect.x+rect.width &&
        rotatedMouseY>rect.y &&
        rotatedMouseY<rect.y+rect.height;
    // draw a dot at the unrotated mouse position
    // green if inside rect, otherwise red
    var hitColor=mouseIsInside?'green':'red';
    dot(mouseX,mouseY,hitColor);
}

body{ background-color: ivory; }
#canvas{border:1px solid red; }

<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
<h4>Clicks inside rect are green, otherwise red.</h4>
<canvas id="canvas" width=512 height=512></canvas>

Answer 2

As you can see on this Codepen i did (to detect 2 rotate rect collide).

You have to check the 2 projections of your point (in my case, the 4 points of a rect) and look if the projections are on the other rect

You have to handle the same thing but only for a point and a rect instead of 2 rects

All projections are not colliding

All projections are colliding

required code for codepen link

Answer 3

Assuming that you know how to check whether a dot is in the rectangle the approach to solution is to rotate and translate everything (dot and rectangle) to "normalized" coordinating system (Cartesian coordinate system that is familiar to us) and then to check it trivially.

For more information you should check Affine transformations. The good link where you could start is

http://www.mathworks.com/discovery/affine-transformation.html?requestedDomain=www.mathworks.com