Interactive Warp Mesh

This example shows usage of Warp node to warp the input image frame. It let’s you interactively change the mesh points to warp the image. After changing the points, user has to press r to restart the pipeline and apply the changes.

User-defined arguments:

  • --mesh_dims - Mesh dimensions (default: 4x4).

  • --resolution - Resolution of the input image (default: 512x512). Width must be divisible by 16.

  • --random - To generate random mesh points (disabled by default).

Originally developed by geaxgx.

Setup

Please run the install script to download all required dependencies. Please note that this script must be ran from git context, so you have to download the depthai-python repository first and then run the script

git clone https://github.com/luxonis/depthai-python.git
cd depthai-python/examples
python3 install_requirements.py

For additional information, please follow installation guide

Demo

https://user-images.githubusercontent.com/18037362/214605914-87cf0404-2d89-478f-9062-2dfb4baa6512.png

Original and warped image

Source code

Python

C++

Also available on GitHub

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#!/usr/bin/env python3
import cv2
import depthai as dai
import numpy as np
import argparse
import re
import sys
from random import randint

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mesh_dims", type=str, default="4x4", help="mesh dimensions widthxheight (default=%(default)s)")
parser.add_argument("-r", "--resolution", type=str, default="512x512", help="preview resolution (default=%(default)s)")
parser.add_argument("-rnd", "--random", action="store_true", help="Generate random initial mesh")
args = parser.parse_args()

# mesh dimensions
match = re.search(r'.*?(\d+)x(\d+).*', args.mesh_dims)
if not match:
    raise Exception(f"Mesh dimensions format incorrect '{args.resolution}'!")
mesh_w = int(match.group(1))
mesh_h = int(match.group(2))

# Preview resolution
match = re.search(r'.*?(\d+)x(\d+).*', args.resolution)
if not match:
    raise Exception(f"Resolution format incorrect '{args.resolution}'!")
preview_w = int(match.group(1))
preview_h = int(match.group(2))
if preview_w % 16 != 0:
    raise Exception(f"Preview width must be a multiple of 16!")

# Create an initial mesh (optionally random) of dimension mesh_w x mesh_h
first_point_x = int(preview_w / 10)
between_points_x = int(4 * preview_w / (5 * (mesh_w - 1)))
first_point_y = int(preview_h / 10)
between_points_y = int(4 * preview_h / (5 * (mesh_h - 1)))
if args.random:
    max_rnd_x = int(between_points_x / 4)
    max_rnd_y = int(between_points_y / 4)
mesh = []
for i in range(mesh_h):
    for j in range(mesh_w):
        x = first_point_x + j * between_points_x
        y = first_point_y + i * between_points_y
        if args.random:
            rnd_x = randint(-max_rnd_x, max_rnd_x)
            if x + rnd_x > 0 and x + rnd_x < preview_w:
                x += rnd_x
            rnd_y = randint(-max_rnd_y, max_rnd_y)
            if y + rnd_y > 0 and y + rnd_y < preview_h:
                y += rnd_y
        mesh.append((x, y))

def create_pipeline(mesh):
    print(mesh)
    # Create pipeline
    pipeline = dai.Pipeline()

    camRgb = pipeline.create(dai.node.ColorCamera)
    camRgb.setPreviewSize(preview_w, preview_h)
    camRgb.setInterleaved(False)
    width = camRgb.getPreviewWidth()
    height = camRgb.getPreviewHeight()

    # Output source
    xout_source = pipeline.create(dai.node.XLinkOut)
    xout_source.setStreamName('source')
    camRgb.preview.link(xout_source.input)
    # Warp source frame
    warp = pipeline.create(dai.node.Warp)
    warp.setWarpMesh(mesh, mesh_w, mesh_h)
    warp.setOutputSize(width, height)
    warp.setMaxOutputFrameSize(width * height * 3)
    camRgb.preview.link(warp.inputImage)

    warp.setHwIds([1])
    warp.setInterpolation(dai.Interpolation.NEAREST_NEIGHBOR)
    # Output warped
    xout_warped = pipeline.create(dai.node.XLinkOut)
    xout_warped.setStreamName('warped')
    warp.out.link(xout_warped.input)
    return pipeline

point_selected = None

def mouse_callback(event, x, y, flags, param):
    global mesh, point_selected, mesh_changed
    if event == cv2.EVENT_LBUTTONDOWN:
        if point_selected is None:
            # Which point is selected ?
            min_dist = 100

            for i in range(len(mesh)):
                dist = np.linalg.norm((x - mesh[i][0], y - mesh[i][1]))
                if dist < 20 and dist < min_dist:
                    min_dist = dist
                    point_selected = i
            if point_selected is not None:
                mesh[point_selected] = (x, y)
                mesh_changed = True

    elif event == cv2.EVENT_LBUTTONUP:
        point_selected = None
    elif event == cv2.EVENT_MOUSEMOVE:
        if point_selected is not None:
            mesh[point_selected] = (x, y)
            mesh_changed = True


cv2.namedWindow("Source")
cv2.setMouseCallback("Source", mouse_callback)

running = True

print("Use your mouse to modify the mesh by clicking/moving points of the mesh in the Source window")
print("Then press 'r' key to restart the device/pipeline")
while running:
    pipeline = create_pipeline(mesh)
    # Connect to device and start pipeline
    with dai.Device(pipeline) as device:
        print("Starting device")
        # Output queue will be used to get the rgb frames from the output defined above
        q_source = device.getOutputQueue(name="source", maxSize=4, blocking=False)
        q_warped = device.getOutputQueue(name="warped", maxSize=4, blocking=False)

        restart_device = False
        mesh_changed = False
        while not restart_device:
            in0 = q_source.get()
            if in0 is not None:
                source = in0.getCvFrame()
                color = (0, 0,255) if mesh_changed else (0,255,0)
                for i in range(len(mesh)):
                    cv2.circle(source, (mesh[i][0], mesh[i][1]), 4, color, -1)
                    if i % mesh_w != mesh_w -1:
                        cv2.line(source, (mesh[i][0], mesh[i][1]), (mesh[i+1][0], mesh[i+1][1]), color, 2)
                    if i + mesh_w < len(mesh):
                        cv2.line(source, (mesh[i][0], mesh[i][1]), (mesh[i+mesh_w][0], mesh[i+mesh_w][1]), color, 2)
                cv2.imshow("Source", source)

            in1 = q_warped.get()
            if in1 is not None:
                cv2.imshow("Warped", in1.getCvFrame())

            key = cv2.waitKey(1)
            if key == ord('r'): # Restart the device if mesh has changed
                if mesh_changed:
                    print("Restart requested...")
                    mesh_changed = False
                    restart_device = True
            elif key == 27 or key == ord('q'): # Exit
                running = False
                break

WIP

Got questions?

Head over to Discussion Forum for technical support or any other questions you might have.