Mesh Warp SVG - I think I've got really close to having it working

Question

I'm trying to make a similar distort function as Illustrators "Free Distort". I have managed to find two scripts that works separately but not together. example warp

1st script: Warps using control points but only the edges. 2nd script: Warps anywhere on the SVG by smudging with the mouse.

I think there is a solution somewhere in these two codes but I haven't been able to find it.

Here is a combination of the two (notice you can smudge and drag the control points but not smudge(warp) by dragging the control points)

To understand what I am talking about please first try to drag the dots and also click on the svg element directly to smudge it.

https://jsfiddle.net/476f0av9/

let brushSize = 300;

const container = document.getElementById('smudge')
const svg = document.getElementById('svg')
const cursor = document.getElementById('cursor')
const warp = new Warp(svg)

const svgPos = container.getBoundingClientRect()
const originX = svgPos.left
const originY = svgPos.top

let pointCount = 0
let smudging = false
let mouseX = 0
let mouseY = 0
let lastMouseX = 0
let lastMouseY = 0
let mouseDeltaX = 0
let mouseDeltaY = 0

////////////////////////////////////////////////////////////////////////

          
          const controlPath = document.getElementById('control-path');
 const width = svg.width.baseVal.value;
          const height = svg.height.baseVal.value;
          
          // Need to interpolate first, so angles remain sharp
       //   warp.interpolate(40);
          
          // Define the number of columns and rows in the grid
          const numColumns = 5;
          const numRows = 5;
          
          // Calculate the grid cell size
          const cellWidth = width / (numColumns - 1);
          const cellHeight = height / (numRows - 1);
          
          // Create the initial control points grid
          const controlPoints = [];
          
          for (let row = 0; row < numRows; row++) {
          
              for (let col = 0; col < numColumns; col++) {
          
                  const x = col * cellWidth;
                  const y = row * cellHeight;
                  
                  controlPoints.push([x, y]);
              }
          }
          
          // Funny things happen when control points are positioned perfectly on other points... buff it out
          const controlBuffer = 0.1;
          
          for (let i = 0; i < controlPoints.length; i++) {
          
              const point = controlPoints[i];
          
              if (point[0] === 0) point[0] -= controlBuffer;
              if (point[1] === 0) point[1] -= controlBuffer;
              if (point[0] === width) point[0] += controlBuffer;
              if (point[1] === height) point[1] += controlBuffer;
          }
          
drawControlShape();
          
 // Function to draw the control shape
   function drawControlShape(element = controlPath, V = controlPoints) {
          
              const path = [];
          
              for (let row = 0; row < numRows; row++) {
          
  const rowPoints = V.slice(row * numColumns, (row + 1) * numColumns);

                  path.push(`M${rowPoints[0][0]} ${rowPoints[0][1]}`);
          
                  for (let col = 1; col < numColumns; col++) {
          
                      path.push(`L${rowPoints[col][0]} ${rowPoints[col][1]}`);
                  }
              }
          
              for (let col = 0; col < numColumns; col++) {
          
                  const colPoints = [];
          
                  for (let row = 0; row < numRows; row++) {
          
                    colPoints.push(V[row * numColumns + col]);
                  }
          
                  path.push(`M${colPoints[0][0]} ${colPoints[0][1]}`);
          
                  for (let row = 1; row < numRows; row++) {
          
                    path.push(`L${colPoints[row][0]} ${colPoints[row][1]}`);
                  }
              }
          
              element.setAttribute('d', path.join(''));
          }
          
       // Add code to draw dots at each control point
          const svgControl = document.getElementById('svg-control');
          
          for (let i = 0; i < controlPoints.length; i++) {
          
            const [x, y] = controlPoints[i];
            const dot = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
          
            dot.setAttribute('cx', x);
            dot.setAttribute('cy', y);
            dot.setAttribute('r', 5);
            dot.setAttribute('fill', 'red');
          
            svgControl.appendChild(dot);
          }
          
          
  var dots = document.querySelectorAll('#svg-control circle');
          
          // Function to initiate dragging
          function startDrag(event) {

            // Store the current dot and its initial position
            const activeDot = event.target;

            let initialX = event.clientX;
            let initialY = event.clientY;
            let offsetX = parseInt(activeDot.getAttribute('cx'));
            let offsetY = parseInt(activeDot.getAttribute('cy'));

            // Get the index of the active dot in the dots NodeList
            const index = Array.from(dots).indexOf(activeDot);

            // Add event listeners for mouse move and mouse up events
            document.addEventListener('mousemove', drag);
            document.addEventListener('mouseup', endDrag);

            // Function to handle dragging
            function drag(event) {

              // Calculate the distance moved by the mouse
              const deltaX = event.clientX - initialX;
              const deltaY = event.clientY - initialY;

              // Update the position of the dot based on the mouse movement
              const newX = offsetX + deltaX;
              const newY = offsetY + deltaY;
              activeDot.setAttribute('cx', newX);
              activeDot.setAttribute('cy', newY);

              // Calculate the row and column indices of the active dot
              const rowIndex = Math.floor(index / numColumns);
              const colIndex = index % numColumns;

              // Update the positions of the affected control points in the grid
              controlPoints[rowIndex * numColumns + colIndex] = [newX, newY];

              // Redraw the control shape
              drawControlShape();

              // Recompute the weights
             // smudge it !
             // warp.transform((v0, V) => reposition(v0, V));
            }

            // Function to end dragging
            function endDrag() {
              // Remove the event listeners for mouse move and mouse up events
              document.removeEventListener('mousemove', drag);
              document.removeEventListener('mouseup', endDrag);
            }
          }
          
          // Add event listeners for mouse interactions on each dot
          dots.forEach(dot => {
            
            dot.addEventListener('mousedown', startDrag);
          });
          
//////////////////////////////////////////////////////////////
window.addEventListener('keydown', function(e)
{
    switch(e.key)
    {
        case 'ArrowUp': { brushSize += 10 } break
        case 'ArrowDown': { brushSize -= 10 }
    }
    
    brushSize = Math.max(10, Math.min(brushSize, 250))
})

window.addEventListener('mousedown', e => smudging = true)
window.addEventListener('mouseup', e => smudging = false)
window.addEventListener('mousemove', function(e)
{
    mouseX = e.clientX;
    mouseY = e.clientY;
    mouseDeltaX = mouseX - lastMouseX
    mouseDeltaY = mouseY - lastMouseY
    lastMouseX = mouseX
    lastMouseY = mouseY
})

function smudge([x, y])
{
    const pointX = x + originX
    const pointY = y + originY
    const deltaX = mouseX - pointX
    const deltaY = mouseY - pointY
    const delta = Math.sqrt(deltaX**2 + deltaY**2)
  

    if(delta <= brushSize)
    {
        x += mouseDeltaX * ((brushSize - delta) / brushSize)
        y += mouseDeltaY * ((brushSize - delta) / brushSize)
    }
    
    return [x, y]
}

function update()
{
  
    cursor.style.transform = `translate(${mouseX}px, ${mouseY}px)`
    cursor.style.fontSize = `${brushSize}px`
  
  
    if(smudging)
    {
        if(pointCount < 4000)
        {
            // warp.preInterpolate(smudge, 8)
            warp.interpolate(8)
        }
            
        pointCount = 0
        warp.transform(function(points)
        {
            pointCount++
            return smudge(points)
        })
        
        mouseDeltaX = (lastMouseX === mouseX ? 0 : mouseDeltaX)
        mouseDeltaY = (lastMouseY === mouseY ? 0 : mouseDeltaY)
    
    }
    
    requestAnimationFrame(update)
}

update()

#svg8,
#svg-control {
position: absolute;
top:0px;
left:0px;
    overflow: visible;
}

 #svg-control {
      position: absolute;
       z-index: 9000000;
      top: 0;
      left: 0;
    }

#control-path {
    fill: none;
    stroke: red;
    stroke-width: 1px;
}

body
{
  padding: 10px;
}

    #smudge {
      position: relative;
    }

    #cursor {
      position: absolute;
    }

 <div id="cursor"></div>
 
 <svg id="svg-control" width="120" height="120">
    <path id="control-path" d="M50 50L100 50L100 100L50 100Z" stroke="black" fill="transparent"/>
  </svg>

<div id="smudge">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 96 105"  height="500px" width="500px" id="svg" >
    
  <g fill="#97C024" stroke="#97C024" stroke-linejoin="round" stroke-linecap="round">
    <path d="M14,40v24M81,40v24M38,68v24M57,68v24M28,42v31h39v-31z" stroke-width="12"/>
    <path d="M32,5l5,10M64,5l-6,10 " stroke-width="2"/>
  </g>
  <path d="M22,35h51v10h-51zM22,33c0-31,51-31,51,0" fill="#97C024"/>
  <g fill="#FFF">
    <circle cx="36" cy="22" r="2"/>
    <circle cx="59" cy="22" r="2"/>
  </g>
</svg>
</div>

<script src="https://cdn.jsdelivr.net/npm/warpjs@1.0.8/dist/warp.js"></script>

Any help would be really appreciated.

Thank you.