How to warp an document image in python correctly?

Question

By using this link, I made the deformed mesh:

inputs = cv2.imread("../datasets/images/0.jpg")
nh, nw = inputs.shape[0]//8, inputs.shape[1]//8

inputs = cv2.resize(inputs, dsize=(nh, nw), interpolation=cv2.INTER_AREA)

mr = nh
mc = nw

xx = np.arange(mr-1, -1, -1)
yy = np.arange(0, mc, 1)
[Y, X] = np.meshgrid(xx, yy)
ms = np.transpose(np.asarray([X.flatten('F'), Y.flatten('F')]), (1,0))

perturbed_mesh = ms
nv = np.random.randint(20) - 1
for k in range(nv):
    #Choosing one vertex randomly
    vidx = np.random.randint(np.shape(ms)[0])
    vtex = ms[vidx, :]
    #Vector between all vertices and the selected one
    xv  = perturbed_mesh - vtex
    #Random movement 
    mv = (np.random.rand(1,2) - 0.5)*20
    hxv = np.zeros((np.shape(xv)[0], np.shape(xv)[1] +1) )
    hxv[:, :-1] = xv
    hmv = np.tile(np.append(mv, 0), (np.shape(xv)[0],1))
    d = np.cross(hxv, hmv)
    d = np.absolute(d[:, 2])
    d = d / (np.linalg.norm(mv, ord=2))
    wt = d

    curve_type = np.random.rand(1)
    if curve_type > 0.3:
        alpha = np.random.rand(1) * 50 + 50
        wt = alpha / (wt + alpha)
    else:
        alpha = np.random.rand(1) + 1
        wt = 1 - (wt / 100 )**alpha
    msmv = mv * np.expand_dims(wt, axis=1)
    perturbed_mesh = perturbed_mesh + msmv

So I got the mesh like:

Then I tried to map the source image pixels onto the generated mesh.

img = cv2.copyMakeBorder(inputs, dh, dh, dw, dw, borderType=cv2.BORDER_CONSTANT, value=(0,0,0))
xs, ys = perturbed_mesh[:, 0], perturbed_mesh[:, 1]
xs = xs.reshape(nh, nw).astype(np.float32)
ys = ys.reshape(nh, nw).astype(np.float32)
dst = cv2.remap(img, xs, ys, cv2.INTER_CUBIC)

plt.imshow(dst)

Finally, I got the result:

But this image have a document on the corner, I can't use it.

How to map the document onto the center of image?

Answer 1

Here is an example of what I did for a perspective warp in Python/OpenCV. It will show you how I achieved the expanded view of the output. Not only did I increase the output size, but I also shifted the output control points. I shifted by +500 px and doubled that to +1000 for the output size.

Input:

No Expand Case:

import numpy as np
import cv2

# read input
img = cv2.imread("building.jpg")

# resize
height,width = 1000,1500
img = cv2.resize(img, (width,height))

# specify conjugate coordinates and shift output on left and top
pts1 = np.float32([[ 250, 0],[1220, 300],[1300, 770],[ 250, 860]])
pts2 = np.float32([[0,0],[width,0],[width,height],[0,height]])

# compute perspective matrix
matrix = cv2.getPerspectiveTransform(pts1,pts2)

print(matrix.shape)
print(matrix)

# convert image to BGRA with opaque alpha
img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)

# do perspective transformation setting area outside input to transparent
# extend output size so extended by 500 all around
imgOutput = cv2.warpPerspective(img, matrix, (width,height), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))

# resize output, since it is too large to post
imgOutput = cv2.resize(imgOutput, (width,height))
    
# save the warped output
cv2.imwrite("building_warped_unexpanded.png", imgOutput)

# show the result
cv2.imshow("result", imgOutput)
cv2.waitKey(0)
cv2.destroyAllWindows()

No Expand Warped Result:

Expanded Case:

import numpy as np
import cv2

# read input
img = cv2.imread("building.jpg")

# resize
height,width = 1000,1500
img = cv2.resize(img, (width,height))

# specify conjugate coordinates and shift output on left and top
pts1 = np.float32([[ 250, 0],[1220, 300],[1300, 770],[ 250, 860]])
pts2 = np.float32([[+500,+500],[width+500,+500],[width+500,height+500],[+500,height+500]])

# compute perspective matrix
matrix = cv2.getPerspectiveTransform(pts1,pts2)

print(matrix.shape)
print(matrix)

# convert image to BGRA with opaque alpha
img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)

# do perspective transformation setting area outside input to transparent
# extend output size so extended by 500 all around
imgOutput = cv2.warpPerspective(img, matrix, (width+1000,height+1000), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))

# resize output, since it is too large to post
imgOutput = cv2.resize(imgOutput, (width,height))
    
# save the warped output
cv2.imwrite("building_warped.jpg", imgOutput)

# show the result
cv2.imshow("result", imgOutput)
cv2.waitKey(0)
cv2.destroyAllWindows()

Expanded Result: