Reputation: 21

Canvas Drawing Mask Not Accurately Captured in HTML5 and JavaScript for Image Inpainting

I'm working on an image inpainting application where users can paint over areas they want to remove on a canvas. The drawn mask should then be sent to the server for processing. However, I'm encountering an issue where the generated mask doesn't match the drawn area accurately, resulting in poor inpainting results.

Here's what I've done so far: HTML Structure:

Here's what I've done so far:
HTML Structure:
`<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Inpaint Image</title>
    <style>
        /* Styles omitted for brevity */
    </style>
</head>
<body>
    <div class="container">
        <h1>Paint on the object you want to remove</h1>
        <canvas id="canvas" width="300" height="200"></canvas>
        <button id="undoButton" disabled>Undo</button>
        <form id="maskForm">
            <input type="hidden" name="maskData" id="maskData">
            <input type="hidden" name="filename" id="filename" value="{{ filename }}">
            <input type="button" value="Remove" onclick="submitMask();" id="removeButton">
        </form>
        <label for="brushSize">Brush Size:</label>
        <input type="range" id="brushSize" name="brushSize" min="1" max="50" value="5">
        <button id="downloadButton" style="display:none;">Download Image</button>
    </div>
    <script>
        const canvas = document.getElementById('canvas');
        const ctx = canvas.getContext('2d');
        const img = new Image();
        img.src = `/static/uploads/{{ filename }}`;
        const undoButton = document.getElementById('undoButton');
        let undoStack = [];
        let mask = [];
        let brushSize = 5;  // Default brush size

        img.onload = () => {
            canvas.width = img.width;
            canvas.height = img.height;
            ctx.drawImage(img, 0, 0);
            mask = new Uint8Array(canvas.width * canvas.height).fill(0);
            saveState();
        };

        let drawing = false;

        function saveState() {
            undoStack.push(canvas.toDataURL());
            undoButton.disabled = false;
        }

        function restoreState() {
            if (undoStack.length > 0) {
                const state = undoStack.pop();
                const img = new Image();
                img.src = state;
                img.onload = () => {
                    ctx.clearRect(0, 0, canvas.width, canvas.height);
                    ctx.drawImage(img, 0, 0);
                    const imgData = ctx.getImageData(0, 0, canvas.width, canvas.height);
                    for (let i = 0; i < imgData.data.length; i += 4) {
                        mask[i / 4] = imgData.data[i] === 255 ? 255 : 0;
                    }
                };
            }
            if (undoStack.length === 0) {
                undoButton.disabled = true;
            }
        }

        canvas.addEventListener('mousedown', () => {
            drawing = true;
            saveState();
        });

        canvas.addEventListener('mouseup', () => {
            drawing = false;
        });

        canvas.addEventListener('mouseout', () => {
            if (drawing) {
                drawing = false;
            }
        });

        canvas.addEventListener('mousemove', (e) => {
            if (!drawing) return;
            const rect = canvas.getBoundingClientRect();
            const x = e.clientX - rect.left;
            const y = e.clientY - rect.top;
            ctx.fillStyle = 'rgba(255, 0, 0, 1.0)';
            ctx.beginPath();
            ctx.arc(x, y, brushSize, 0, 2 * Math.PI);
            ctx.fill();
            updateMask(x, y, brushSize);
        });

        document.getElementById('brushSize').addEventListener('input', (e) => {
            brushSize = parseInt(e.target.value);
        });

        function updateMask(x, y, brushSize) {
            const rect = canvas.getBoundingClientRect();
            const scaleX = canvas.width / rect.width;
            const scaleY = canvas.height / rect.height;
            const scaledX = Math.floor(x * scaleX);
            const scaledY = Math.floor(y * scaleY);
            const brushRadius = Math.floor(brushSize * scaleX / 2);

            for (let i = -brushRadius; i <= brushRadius; i++) {
                for (let j = -brushRadius; j <= brushRadius; j++) {
                    const dist = Math.sqrt(i * i + j * j);
                    if (dist <= brushRadius) {
                        const maskX = scaledX + i;
                        const maskY = scaledY + j;
                        if (maskX >= 0 && maskX < canvas.width && maskY >= 0 && maskY < canvas.height) {
                            mask[maskY * canvas.width + maskX] = 255;
                        }
                    }
                }
            }
        }

        function submitMask() {
            const maskData = Array.from(mask).join(',');
            const filename = document.getElementById('filename').value;
            canvas.classList.add('blinking');
            fetch('/inpaint_ajax', {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({ maskData, filename }),
            })
            .then(response => response.blob())
            .then(blob => {
                const url = URL.createObjectURL(blob);
                img.src = url;
                img.onload = () => {
                    ctx.clearRect(0, 0, canvas.width, canvas.height);
                    canvas.width = img.width;
                    canvas.height = img.height;
                    ctx.drawImage(img, 0, 0);
                    mask = new Uint8Array(canvas.width * canvas.height).fill(0);
                    canvas.classList.remove('blinking');
                    document.getElementById('downloadButton').style.display = 'block';
                    document.getElementById('downloadButton').onclick = () => downloadImage(url);
                };
            })
            .catch(error => console.error('Error:', error));
        }

        function downloadImage(url) {
            const a = document.createElement('a');
            a.href = url;
            a.download = 'processed_image.png';
            document.body.appendChild(a);
            a.click();
            document.body.removeChild(a);
        }

        undoButton.addEventListener('click', restoreState);
    </script>
</body>
</html>

Backend Processing (Flask):

`@app.route('/inpaint_ajax', methods=['POST'])
 def inpaint_ajax():
   data = request.get_json()
   filename = data['filename']
   mask_data = data['maskData']

   image_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
   img = Image.open(image_path).convert('RGB')
   original_size = img.size

  fixed_size = app.config['FIXED_SIZE']

  image_tensor = preprocess_image(image_path, fixed_size)
  mask_tensor = preprocess_mask(mask_data, original_size[::-1], fixed_size)

  lama_model = load_lama_model(app.config['MODEL_PATH'])
  with torch.no_grad():
      inpainted_tensor = lama_model(image_tensor, mask_tensor)

  output_path = os.path.join(app.config['OUTPUT_FOLDER'], filename)
  postprocess_and_save(inpainted_tensor, output_path, original_size)

  shutil.copy(output_path, image_path)

  mask_array = np.array(mask_data.split(','), dtype=np.uint8).reshape(original_size[::-1])
  mask_image_path = os.path.join(app.config['UPLOAD_FOLDER'], f'mask_{filename}')
  cv2.imwrite(mask_image_path, mask_array)

  return send_from_directory(app.config['OUTPUT_FOLDER'], filename)

Despite implementing the above code, the mask generated on the server doesn't match the area drawn by the user. The mask seems to miss parts or doesn't align with the brush strokes accurately.

Upvotes: -1

Answers (1)

Ashmit Singh

Reputation: 21

By incorporating both scaling factors, the mask generation now accurately reflects the drawn area.

function updateMask(x, y, brushSize) {
  const rect = canvas.getBoundingClientRect();
  const scaleX = canvas.width / rect.width;
  const scaleY = canvas.height / rect.height;


  const brushRadius = Math.floor((brushSize * scaleX + brushSize * scaleY) / 2);

  const startX = Math.max(0, Math.floor((x - brushRadius) * scaleX));
  const startY = Math.max(0, Math.floor((y - brushRadius) * scaleY));
  const endX = Math.min(canvas.width, Math.ceil((x + brushRadius) * scaleX));
  const endY = Math.min(canvas.height, Math.ceil((y + brushRadius) * scaleY));

  for (let i = startX; i < endX; i++) {
      for (let j = startY; j < endY; j++) {
          const dx = i / scaleX - x;
          const dy = j / scaleY - y;
          if (Math.sqrt(dx * dx + dy * dy) <= brushRadius) {
              mask[j * canvas.width + i] = 255;
          }
      }
  }
}

Upvotes: 0

Canvas Drawing Mask Not Accurately Captured in HTML5 and JavaScript for Image Inpainting

Answers (1)

Related Questions