How do I rotate, scale and translate on Html5 Canvas?

Question

I've tried for the last few days without too much success to rotate, scale and translate shapes on the canvas. I've read everything I could find on internet about similar issues but still I cannot seem to be able to adapt it to my own problem.

If everything is drawn on the same scale, I can still drag and drop. If I rotate the shapes, then the mouseOver is messed up since the world coordinates don't correspond anymore with the shape coordinates. If I scale, then it's impossible to select any shape. I look at my code and do not understand what I'm doing wrong.

I read some really nice and detailed stackoverflow solutions to similar problems. For example, user @blindman67 made a suggestion of using a setTransform helper and a getMouseLocal helper for getting the coordinates of the mouse relative to the transformed shape.

inverse transform matrix

I spent some time with that and could not fix my issue. Here is an example of what I tried. Any suggestion is appreciated.

const canvas = document.getElementById('canvas');
const ctx = canvas.getContext('2d');

canvas.width = window.innerWidth - 40;
canvas.height = window.innerHeight - 60;
const canvasBounding = canvas.getBoundingClientRect();
const offsetX = canvasBounding.left;
const offsetY = canvasBounding.top;

let scale = 1;
let selectedShape = '';
let startX = 0;
let startY = 0;
let endX = 0;
let endY = 0;
let mouseIsDown = false;
let mouseIsMovingShape = false;
let selectedTool = 'SELECT';

let shapes = {};

const selectButton = document.getElementById('select');
const rectangleButton = document.getElementById('rectangle');

canvas.addEventListener('mousedown', canvasMouseDown);
canvas.addEventListener('mouseup', canvasMouseUp);
canvas.addEventListener('mousemove', canvasMouseMove);

function canvasMouseDown(e) {
  e.preventDefault();
  const mouseX = e.clientX - offsetX;
  const mouseY = e.clientY - offsetY;
  startX = mouseX;
  startY = mouseY;
  mouseIsDown = true;
  selectedShape = '';
  if (selectedTool === 'SELECT') {
    for (const shapeId in shapes) {
      const shape = shapes[shapeId];
      if (shape.mouseIsOver(mouseX, mouseY)) {
        selectedShape = shape.id;
        shapes[shape.id].isSelected = true;
      } else {
        shapes[shape.id].isSelected = false;
      }
    }
  }
  draw();
}

function canvasMouseUp(e) {
  e.preventDefault();
  const mouseX = e.clientX - offsetX;
  const mouseY = e.clientY - offsetY;
  endX = mouseX;
  endY = mouseY;
  mouseIsDown = false;
  const tooSmallShape = Math.abs(endX) - startX < 1 || Math.abs(endY) - startY < 1;
  if (tooSmallShape) {
    return;
  }
  if (selectedTool === 'RECTANGLE') {
    const newShape = new Shape(selectedTool.toLowerCase(), startX, startY, endX, endY);
    shapes[newShape.id] = newShape;
    selectedShape = '';
    setActiveTool('SELECT');
  }
  draw();
}

function canvasMouseMove(e) {
  e.preventDefault();
  const mouseX = e.clientX - offsetX;
  const mouseY = e.clientY - offsetY;
  const dx = e.movementX;
  const dy = e.movementY;
  if (mouseIsDown) {
    draw();
    if (selectedTool === 'SELECT' && selectedShape !== '') {
      const shape = shapes[selectedShape];
      shape.x += dx;
      shape.y += dy;
    }
    if (selectedTool === 'RECTANGLE') {
      drawShapeGhost(mouseX, mouseY);
    }
  }
}

function draw() {
  clear();
  for (const shapeId in shapes) {
    const shape = shapes[shapeId];
    shape.drawShape(ctx);
  }
}

function clear() {
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  ctx.fillStyle = 'rgba(255, 255, 255, 1)';
  ctx.fillRect(0, 0, canvas.width, canvas.height)
}

function drawShapeGhost(x, y) {
  ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)';
  ctx.strokeRect(startX, startY, x - startX, y - startY);
  ctx.stroke();
}

function setActiveTool(tool) {
  selectedTool = tool;
  if (tool === 'RECTANGLE') {
    rectangleButton.classList.add('active');
    selectButton.classList.remove('active');
    selectedTool = tool;
  }
  if (tool === 'SELECT') {
    rectangleButton.classList.remove('active');
    selectButton.classList.add('active');
    selectedTool = tool;
  }
}

function degreesToRadians(degrees) {
  return (Math.PI * degrees) / 180;
};


class Shape {
  constructor(shapeType, startX, startY, endX, endY, fill, stroke) {
    this.id = shapeType + Date.now();
    this.type = shapeType;
    this.x = startX;
    this.y = startY;
    this.width = Math.abs(endX - startX);
    this.height = Math.abs(endY - startY);
    this.fill = fill || 'rgba(149, 160, 178, 0.8)';
    this.stroke = stroke || 'rgba(0, 0, 0, 0.8)';
    this.rotation = 0;
    this.isSelected = false;
    this.scale = 1;
  }

  drawShape(ctx) {
    switch (this.type) {
      case 'rectangle':
        this._drawRectangle(ctx);
        break;
    }
  }

  _drawRectangle(ctx) {
    ctx.save();
    ctx.scale(this.scale, this.scale);
    ctx.translate(this.x + this.width / 2, this.y + this.height / 2);
    ctx.rotate(degreesToRadians(this.rotation));

    if (this.fill) {
      ctx.fillStyle = this.fill;
      ctx.fillRect(-this.width / 2, -this.height / 2, this.width, this.height);
    }

    if (this.stroke !== null) {
      ctx.strokeStyle = this.stroke;
      ctx.strokeWidth = 1;
      ctx.strokeRect(-this.width / 2, -this.height / 2, this.width, this.height);
      ctx.stroke();
    }

    if (this.isSelected) {
      ctx.strokeStyle = 'rgba(254, 0, 0, 1)';
      ctx.strokeWidth = 1;
      ctx.strokeRect(-this.width / 2, -this.height / 2, this.width, this.height)
      ctx.stroke();
      ctx.closePath();
    }
    ctx.restore();
  }

  mouseIsOver(mouseX, mouseY) {
    if (this.type === 'rectangle') {
      return (mouseX > this.x && mouseX < this.x + this.width && mouseY > this.y && mouseY < this.y + this.height);
    }
  }
}

const menu = document.getElementById('menu');
const rotation = document.getElementById('rotation');
const scaleSlider = document.getElementById('scale');

menu.addEventListener('click', onMenuClick);
rotation.addEventListener('input', onRotationChange);
scaleSlider.addEventListener('input', onScaleChange);

function onMenuClick(e) {
  const tool = e.target.dataset.tool;
  if (tool && tool === 'RECTANGLE') {
    rectangleButton.classList.add('active');
    selectButton.classList.remove('active');
    selectedTool = tool;
  }
  if (tool && tool === 'SELECT') {
    rectangleButton.classList.remove('active');
    selectButton.classList.add('active');
    selectedTool = tool;
  }
}

function onRotationChange(e) {
  if (selectedShape !== '') {
    shapes[selectedShape].rotation = e.target.value;
    draw();
  }
}

function onScaleChange(e) {
  scale = e.target.value;
  for (const shapeId in shapes) {
    const shape = shapes[shapeId];
    shape.scale = scale;
  }
  draw();
}


function setTransform(ctx, x, y, scaleX, scaleY, rotation) {
  const xDx = Math.cos(rotation);
  const xDy = Math.sin(rotation);
  ctx.setTransform(xDx * scaleX, xDy * scaleX, -xDy * scaleY, xDx * scaleY, x, y);
}

function getMouseLocal(mouseX, mouseY, x, y, scaleX, scaleY, rotation) {
  const xDx = Math.cos(rotation);
  const xDy = Math.sin(rotation);

  const cross = xDx * scaleX * xDx * scaleY - xDy * scaleX * (-xDy) * scaleY;

  const ixDx = (xDx * scaleY) / cross;
  const ixDy = (-xDy * scaleX) / cross;
  const iyDx = (xDy * scaleY) / cross;
  const iyDy = (xDx * scaleX) / cross;

  mouseX -= x;
  mouseY -= y;

  const localMouseX = mouseX * ixDx + mouseY * iyDx;
  const localMouseY = mouseX * ixDy + mouseY * iyDy;

  return {
    x: localMouseX,
    y: localMouseY,
  }
}

function degreesToRadians(degrees) {
  return (Math.PI * degrees) / 180
};

function radiansToDegrees(radians) {
  return radians * 180 / Math.PI
};

let timer;

function debounce(fn, ms) {
  clearTimeout(timer);
  timer = setTimeout(() => fn(), ms);
}

canvas {
  margin-top: 1rem;
  border: 1px solid black;
}

button {
  border: 1px solid #adadad;
  background-color: transparent;
}

.active {
  background-color: lightblue;
}

.menu {
  display: flex;
}

.d-flex {
  display: flex;
  margin-left: 2rem;
}

.rotation input {
  margin-left: 1rem;
  max-width: 50px;
}

<div id="menu" class="menu">
  <button id="select" data-tool="SELECT">select</button>
  <button id="rectangle" data-tool="RECTANGLE">rectangle</button>
  <div class="d-flex">
    <label for="rotation">rotation </label>
    <input type="number" id="rotation" value="0">
  </div>
  <div class="d-flex">
    <label for="scale">scale </label>
    <input type="range" step="0.1" min="0.1" max="10" value="1" name="scale" id="scale">
  </div>
</div>
<canvas id="canvas"></canvas>

Answer 1

If I have time tomorrow I will try to implement the following to your code but I can provide you with a working example of how to get mouse collision precision on a rotated rectangle. I had the same struggle with this and finally found a good explanation and code that I was able to get to work. Check out this website

Now for my own implementation I did not use the method on that website to get my vertices. As you'll see in my code I have a function called updateCorners() in my Square class. I also have objects called this.tl.x and this.tl.y (for each corner).

The formulas are what I use to get vertices of a translated and rotated rectangle and the corner objects are what are used to determine collision. From there I used the distance() function (Pythagorean theorem), the triangleArea() function, and then the clickHit() function which I renamed to collision() and changed some things.

Example in the snippet below

let canvas = document.getElementById("canvas");
let ctx = canvas.getContext("2d");
canvas.width = 400;
canvas.height = 400;
let shapes = [];
let mouse = {
  x: null,
  y: null
}
canvas.addEventListener('mousemove', e => {
  mouse.x = e.x - canvas.getBoundingClientRect().x;
  mouse.y = e.y - canvas.getBoundingClientRect().y;
  
})

class Square {
  constructor(x, y, w, h, c) {
    this.x = x;
    this.y = y;
    this.w = w;
    this.h = h;
    this.c = c;
    this.a = 0;
    this.r = this.a * (Math.PI/180);
    this.cx = this.x + this.w/2;
    this.cy = this.y + this.h/2;
    //used to track corners
    this.tl = {x: 0, y: 0};
    this.tr = {x: 0, y: 0};
    this.br = {x: 0, y: 0};
    this.bl = {x: 0, y: 0};
  }
  draw() {
    ctx.save();
    ctx.translate(this.x, this.y)
    ctx.rotate(this.r);
    ctx.fillStyle = this.c;
    ctx.fillRect(-this.w/2,-this.h/2,this.w,this.h);
    ctx.restore();
  }
  updateCorners() {
    this.a += 0.1
    this.r = this.a * (Math.PI/180);
    let cos = Math.cos(this.r);
    let sin = Math.sin(this.r)
    //updates Top Left Corner
    this.tl.x = (this.x-this.cx)*cos - (this.y-this.cy)*sin+(this.cx-this.w/2);
    this.tl.y = (this.x-this.cx)*sin + (this.y-this.cy)*cos+(this.cy-this.h/2)
    //updates Top Right Corner
    this.tr.x = ((this.x+this.w)-this.cx)*cos - (this.y-this.cy)*sin+(this.cx-this.w/2)
    this.tr.y = ((this.x+this.w)-this.cx)*sin + (this.y-this.cy)*cos+(this.cy-this.h/2)
    //updates Bottom Right Corner
    this.br.x = ((this.x+this.w)-this.cx)*cos - ((this.y+this.h)-this.cy)*sin+(this.cx-this.w/2)
    this.br.y = ((this.x+this.w)-this.cx)*sin + ((this.y+this.h)-this.cy)*cos+(this.cy-this.h/2)
    //updates Bottom Left Corner
    this.bl.x = (this.x-this.cx)*cos - ((this.y+this.h)-this.cy)*sin+(this.cx-this.w/2)
    this.bl.y = (this.x-this.cx)*sin + ((this.y+this.h)-this.cy)*cos+(this.cy-this.h/2)
  }
}
let square1 = shapes.push(new Square(250, 70, 25, 25, 'red'));
let square2 = shapes.push(new Square(175,210, 100, 50, 'blue'));
let square3 = shapes.push(new Square(50,100, 30, 50, 'purple'));
let square4 = shapes.push(new Square(140,120, 120, 20, 'pink'));

//https://joshuawoehlke.com/detecting-clicks-rotated-rectangles/
//pythagorean theorm using built in javascript hypot
function distance(p1, p2) {
    return Math.hypot(p1.x-p2.x, p1.y-p2.y);
}

//Heron's formula used to determine area of triangle
//in the collision() function we will break the rectangle into triangles
function triangleArea(d1, d2, d3) {
    var s = (d1 + d2 + d3) / 2;
    return Math.sqrt(s * (s - d1) * (s - d2) * (s - d3));
}

function collision(mouse, rect) {
  //area of rectangle
    var rectArea = Math.round(rect.w * rect.h);
    // Create an array of the areas of the four triangles
    var triArea = [
        // mouse posit checked against tl-tr
        triangleArea(
            distance(mouse, rect.tl),
            distance(rect.tl, rect.tr),
            distance(rect.tr, mouse)
        ),
        // mouse posit checked against tr-br
        triangleArea(
            distance(mouse, rect.tr),
            distance(rect.tr, rect.br),
            distance(rect.br, mouse)
        ),
        // mouse posit checked against tr-bl
        triangleArea(
            distance(mouse, rect.br),
            distance(rect.br, rect.bl),
            distance(rect.bl, mouse)
        ),
        // mouse posit checked against bl-tl
        triangleArea(
            distance(mouse, rect.bl),
            distance(rect.bl, rect.tl),
            distance(rect.tl, mouse)
        )
    ];
    let triArea2 = Math.round(triArea.reduce(function(a,b) { return a + b; }, 0));
    if (triArea2 > rectArea) {
        return false;
    }
    return true;
}


function animate() {
  ctx.clearRect(0,0,canvas.width,canvas.height);
  ctx.fillStyle = 'black';
  ctx.fillText('x: '+mouse.x+',y: '+mouse.y, 50, 50);
  for (let i=0; i< shapes.length; i++) {
    shapes[i].draw();
    shapes[i].updateCorners();
    if (collision(mouse, shapes[i])) {
    shapes[i].c = 'red';
  } else {
    shapes[i].c = 'green'
  }
  }
  requestAnimationFrame(animate)
}
animate();

<canvas id="canvas"></canvas>

I'm sure there's many other ways to do this but this is what I was able to understand and get to work. I haven't really messed with scale so I can't help much there.

UPDATE: Here is a snippet using the method you wanted. Now you can rotate, scale, and translate and still click inside the shape. Be aware I changed your mouse to a global mouse object vice making it a variable in every mouse... function.

const canvas = document.getElementById("canvas");
const ctx = canvas.getContext("2d");

canvas.width = window.innerWidth - 40;
canvas.height = window.innerHeight - 60;
const canvasBounding = canvas.getBoundingClientRect();
const offsetX = canvasBounding.left;
const offsetY = canvasBounding.top;

let scale = 1;
let selectedShape = "";
let startX = 0;
let startY = 0;
let endX = 0;
let endY = 0;
let mouseIsDown = false;
let mouseIsMovingShape = false;
let selectedTool = "SELECT";
let localMouse = { x: null, y: null };
let mouse = { x: null, y: null };
let shapes = {};

const selectButton = document.getElementById("select");
const rectangleButton = document.getElementById("rectangle");

canvas.addEventListener("mousedown", canvasMouseDown);
canvas.addEventListener("mouseup", canvasMouseUp);
canvas.addEventListener("mousemove", canvasMouseMove);

function canvasMouseDown(e) {
  e.preventDefault();
  mouse.x = e.clientX - offsetX;
  mouse.y = e.clientY - offsetY;
  startX = mouse.x;
  startY = mouse.y;
  mouseIsDown = true;
  selectedShape = "";
  if (selectedTool === "SELECT") {
    for (const shapeId in shapes) {
      const shape = shapes[shapeId];
      if (shape.mouseIsOver()) {
        selectedShape = shape.id;
        shapes[shape.id].isSelected = true;
      } else {
        shapes[shape.id].isSelected = false;
      }
    }
  }
  draw();
}

function canvasMouseUp(e) {
   e.preventDefault();
  mouse.x = e.clientX - offsetX;
  mouse.y = e.clientY - offsetY;
  endX = mouse.x;
  endY = mouse.y;
  mouseIsDown = false;
  const tooSmallShape =
    Math.abs(endX) - startX < 1 || Math.abs(endY) - startY < 1;
  if (tooSmallShape) {
    return;
  }
  if (selectedTool === "RECTANGLE") {
    const newShape = new Shape(
      selectedTool.toLowerCase(),
      startX,
      startY,
      endX,
      endY
    );
    shapes[newShape.id] = newShape;
    selectedShape = "";
    setActiveTool("SELECT");
  }
  draw();
}

function canvasMouseMove(e) {
  e.preventDefault();
  mouse.x = e.clientX - offsetX;
  mouse.y = e.clientY - offsetY;
  const dx = e.movementX;
  const dy = e.movementY;
  if (mouseIsDown) {
    draw();
    if (selectedTool === "SELECT" && selectedShape !== "") {
      const shape = shapes[selectedShape];
      shape.x += dx;
      shape.y += dy;
    }
    if (selectedTool === "RECTANGLE") {
      drawShapeGhost(mouse.x, mouse.y);
    }
  }
}

function draw() {
  clear();
  for (const shapeId in shapes) {
    const shape = shapes[shapeId];
    shape.drawShape(ctx);
  }
}

function clear() {
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  ctx.fillStyle = "rgba(255, 255, 255, 1)";
  ctx.fillRect(0, 0, canvas.width, canvas.height);
}

function drawShapeGhost(x, y) {
  ctx.strokeStyle = "rgba(0, 0, 0, 0.5)";
  ctx.strokeRect(startX, startY, x - startX, y - startY);
  ctx.stroke();
}

function setActiveTool(tool) {
  selectedTool = tool;
  if (tool === "RECTANGLE") {
    rectangleButton.classList.add("active");
    selectButton.classList.remove("active");
    selectedTool = tool;
  }
  if (tool === "SELECT") {
    rectangleButton.classList.remove("active");
    selectButton.classList.add("active");
    selectedTool = tool;
  }
}

function degreesToRadians(degrees) {
  return (Math.PI * degrees) / 180;
}

class Shape {
  constructor(shapeType, startX, startY, endX, endY, fill, stroke) {
    this.id = shapeType + Date.now();
    this.type = shapeType;
    this.x = startX;
    this.y = startY;
    this.width = Math.abs(endX - startX);
    this.height = Math.abs(endY - startY);
    this.fill = fill || "rgba(149, 160, 178, 0.8)";
    this.stroke = stroke || "rgba(0, 0, 0, 0.8)";
    this.rotation = 0;
    this.isSelected = false;
    this.scale = { x: 1, y: 1 };
  }

  drawShape(ctx) {
    switch (this.type) {
      case "rectangle":
        this._drawRectangle(ctx);
        break;
    }
  }

  _drawRectangle(ctx) {
    ctx.save();
    setTransform(
      this.x + this.width / 2,
      this.y + this.height / 2,
      this.scale.x,
      this.scale.y,
      degreesToRadians(this.rotation)
    );

    if (this.fill) {
      ctx.fillStyle = this.fill;
      ctx.fillRect(-this.width / 2, -this.height / 2, this.width, this.height);
    }

    if (this.stroke !== null) {
      ctx.strokeStyle = this.stroke;
      ctx.strokeWidth = 1;
      ctx.strokeRect(
        -this.width / 2,
        -this.height / 2,
        this.width,
        this.height
      );
      ctx.stroke();
    }

    if (this.isSelected) {
      ctx.strokeStyle = "rgba(254, 0, 0, 1)";
      ctx.strokeWidth = 1;
      ctx.strokeRect(
        -this.width / 2,
        -this.height / 2,
        this.width,
        this.height
      );
      ctx.stroke();
      ctx.closePath();
    }
    ctx.restore();
  }
  mouseIsOver() {
    localMouse = getMouseLocal(
      mouse.x,
      mouse.y,
      this.x + this.width / 2,
      this.y + this.height / 2,
      this.scale.x,
      this.scale.y,
      degreesToRadians(this.rotation)
    );
    if (this.type === "rectangle") {
      if (
        localMouse.x > 0 - this.width / 2 &&
        localMouse.x < 0 + this.width / 2 &&
        localMouse.y < 0 + this.height / 2 &&
        localMouse.y > 0 - this.height / 2
      ) {
        return true;
      }
    }
  }
}

const menu = document.getElementById("menu");
const rotation = document.getElementById("rotation");
const scaleSlider = document.getElementById("scale");

menu.addEventListener("click", onMenuClick);
rotation.addEventListener("input", onRotationChange);
scaleSlider.addEventListener("input", onScaleChange);

function onMenuClick(e) {
  const tool = e.target.dataset.tool;
  if (tool && tool === "RECTANGLE") {
    rectangleButton.classList.add("active");
    selectButton.classList.remove("active");
    selectedTool = tool;
  }
  if (tool && tool === "SELECT") {
    rectangleButton.classList.remove("active");
    selectButton.classList.add("active");
    selectedTool = tool;
  }
}

function onRotationChange(e) {
  if (selectedShape !== "") {
    shapes[selectedShape].rotation = e.target.value;
    draw();
  }
}

function onScaleChange(e) {
  scale = e.target.value;
  for (const shapeId in shapes) {
    const shape = shapes[shapeId];
    shape.scale.x = scale;
    shape.scale.y = scale;
  }
  draw();
}

function setTransform(x, y, sx, sy, rotate) {
  var xdx = Math.cos(rotate); // create the x axis
  var xdy = Math.sin(rotate);
  ctx.setTransform(xdx * sx, xdy * sx, -xdy * sy, xdx * sy, x, y);
}

function getMouseLocal(mouseX, mouseY, x, y, sx, sy, rotate) {
  var xdx = Math.cos(rotate); // create the x axis
  var xdy = Math.sin(rotate);
  var cross = xdx * sx * xdx * sy - xdy * sx * -xdy * sy;
  var ixdx = (xdx * sy) / cross; // create inverted x axis
  var ixdy = (-xdy * sx) / cross;
  var iydx = (xdy * sy) / cross; // create inverted y axis
  var iydy = (xdx * sx) / cross;

  mouseX -= x;
  mouseY -= y;

  var localMouseX = mouseX * ixdx + mouseY * iydx;
  var localMouseY = mouseX * ixdy + mouseY * iydy;

  return { x: localMouseX, y: localMouseY };
}

function radiansToDegrees(radians) {
  return (radians * 180) / Math.PI;
}

let timer;

function debounce(fn, ms) {
  clearTimeout(timer);
  timer = setTimeout(() => fn(), ms);
}

canvas {
  margin-top: 1rem;
  border: 1px solid black;
}

button {
  border: 1px solid #adadad;
  background-color: transparent;
}

.active {
  background-color: lightblue;
}

.menu {
  display: flex;
}

.d-flex {
  display: flex;
  margin-left: 2rem;
}

.rotation input {
  margin-left: 1rem;
  max-width: 50px;
}

<div id="menu" class="menu">
  <button id="select" data-tool="SELECT">select</button>
  <button id="rectangle" data-tool="RECTANGLE">rectangle</button>
  <button id="triangle" data-tool="TRIANGLE">triangle</button>
  <div class="d-flex">
    <label for="rotation">rotation </label>
    <input type="number" id="rotation" value="0">
  </div>
  <div class="d-flex">
    <label for="scale">scale </label>
    <input type="range" step="0.1" min="0.1" max="10" value="1" name="scale" id="scale">
  </div>
</div>
<canvas id="canvas"></canvas>