How to extract arbitrary background rectangular patches from an image with many different foregrounds

Question

• I'm working on the FDDB-DataSet to extract the faces, and the same count of backgrounds from the same image.

• I was able to get the faces(from the coordinates of the annotation).

• My problem is that: I want to crop bacground patches with the same number of the faces(not necessarily the same size), the constraints on the backgrounds is :

1. The size of any background image should be no less than 20 x 20 pixels.
2. The maximum overlapping ratio with the faces(any of them)(IOU: Intersection over Union) should not exceed 20% (0.2).
3. The count of the background patches should be the same as the count of the faces in the same image.

Sample Image:

Annotation Data:

[Rmax Rmin theta Xc Yc]

[[ 52  38   1 154  72]
 [ 57  38  -2 368 103]
 [ 14   9  -2  11  64]
 [ 11   8   1  29  16]
 [ 10   6   1  56  61]
 [ 10   6  -2  68  66]
 [ 14   9   1  46 126]
 [ 15  11   1  21 192]
 [ 22  12   1  11 157]
 [ 13   9   1 267 133]
 [ 19  13   1 312 186]
 [ 12   9   1 300  19]
 [ 11   9  -2 334 139]
 [ 14  10  -2 437  87]
 [ 11   8   1 232  27]]

My Code is:

#--------------------
# Import Libraies
#====================
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2

datasetPath = "/home/myPath" 
#---------------------------------------------
# Extract faces, and Backgrounds from an image
#=============================================
def extData(imgPath, annots, foldPath, index):
    '''
    Function to Extract faces, and Backgrounds from an image.
    
    Parameters:
    @Param: imgPath : the specified image path(inside the DB).
    @Param: annots  : nd-array with shape m x 5, : m num of the detected faces in the image.
            (5): max_radius | min_radius | angle | center_x | center_y
    @Param: foldPath : the fold path where to save the extracted images.
    @Param: index : a number to start naming the faces, backgrounds from it.
    
    saves: m nd-array (m Faces), and m nd-array (m Background) in the foldPath
    '''
    fullImagePath = os.path.join(datasetPath, imgPath)
    img = cv2.imread(fullImagePath)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # Create Faces
    facesList = []
    for row in range(annots.shape[0]):
        # Initialize the Variables
        xc = annots[row][3]
        yc = annots[row][4]
        Rmax = annots[row][0] 
        Rmin =  annots[row][1]
        theta =  annots[row][2]

        # Rectangle Borders 
        # represents the top left corner of rectangle 
        x0 = math.floor(xc - Rmin * math.sin(theta))
        y0 = math.floor(yc - Rmax * math.sin(theta)) 
        # represents the bottom right corner of rectangle 
        x1 = math.floor(xc + Rmin * math.sin(theta))
        y1 = math.floor(yc + Rmax * math.sin(theta))
        # Crop the face in rectangular window
        face = img[y0:y1, x0:x1,:]
        # store the coordinates of the face
        facesList.append([x0, x1, y0, y1])
        # make a directory to save the faces inside.
        os.mkdir(os.path.join(foldPath,"/face/"))
        cv2.imwrite(os.path.join(foldPath,"/face/", "face_{}".format(str(index+row)),face))
    
    # Create Backgrounds
    for row, face in enumerate(facesList):

        # background = img[xb1:xb2,yb1:yb2,:]
        # make a directory to save the backgrounds inside.
        os.mkdir(os.path.join(foldPath,"/background/"))
        # cv2.imwrite(os.path.join(foldPath,"/background/", "background_{}".format(str(index+row)),background))
#---------------------------------------------

Answer 1

You can use cv2.distanceTransform to compute the L1 distances from the faces, and sample rect-centers according to their distance, thus ensuring the crops will not overlap with "faces":

import numpy as np

img = cv2.imread(fullImagePath)
# create a mask with zeros in "faces" and bonudary
mask = np.zeros(img.shape[:2], dtype=np.uint8)
mask[1:-1, 1:-1] = 1
for row in range(annots.shape[0]):
  # Initialize the Variables
  xc = annots[row][3]
  yc = annots[row][4]
  Rmax = annots[row][0]
  Rmin =  annots[row][1]
  theta =  annots[row][2]
  # in case of rotation angles more than 45 degrees swap radius
  if(theta > 45):
    R = Rmax
    Rmax = Rmin
    Rmin = R
  # Rectangle Borders
  # represents the top left corner of rectangle
  st = math.sin(theta)
  st = st if st > 0 else -st
  x0 = math.floor(xc - Rmin * st)
  y0 = math.floor(yc - Rmax * st)
  # represents the bottom right corner of rectangle
  x1 = math.floor(xc + Rmin * st)
  y1 = math.floor(yc + Rmax * st)
  # Crop the face in rectangular window
  mask[y0:y1, x0:x1] = 0

# once we have a map we can compute the distance of each non-face pixel to the nearest face
dst = cv2.distanceTransform(mask, cv2.DIST_L1, 3)

# pixels that are closer than 10 pixels (20//2) to a face, cannot be considered as good candidates. If you allow for IoU > 0 this can be relaxed a little.
dst[dst<10] = 0

# linear indices of pixels
idx = np.arange(np.prod(img.shape[:2]))

# sample centers
centers = np.random.choice(idx, size=annots.shape[0], replace=False, p=dst.flatten()/dst.sum())

# create the rectangles
windows = []
for i, c in enumerate(centers):
  r = int(np.floor(dst.flat[c]))
  r = np.random.choice(range(10,r))  # sample possible R from 10 to max possible
  [y, x] = np.unravel_index(c, img.shape[:2])
  windows.append((y-r, x-r, y+r, x+r))

An example of the resulting windows is shown here: