Welcome to DOFBOT-Ultra repository

Facial Detection

Please ensure that the web control program backend process is not running in the background. To close it, refer to the [Get Started Quickly - 2. Access the Terminal] section in [Disable the Auto-Start Web Control Backend Process]. After closing, please restart DofbotUltra!!!

Use MediaPipe Face Mesh to detect facial keypoint mesh (468 keypoints) in real-time, drawing facial contours on both the original image and a black background. Supports simultaneous detection of two faces.

1、Startup

1.1、Startup

• Enter the following command to start the program

xxxxxxxxxx
ros2 run dofbot_mediapipe 04_FaceMesh.py

Press the q key in the image window or Ctrl+C in the terminal to exit the program.

1.2、Source Code

Raspberry Pi source path:

xxxxxxxxxx
~/dofbot_ultra_ws/src/dofbot_mediapipe/scripts/04_FaceMesh.py

RDK X5 source path:

xxxxxxxxxx
~/dofbot_ultra_ws/src/2d/dofbot_mediapipe/scripts/04_FaceMesh.py

#!/usr/bin/env python3
# encoding: utf-8
import cv2 as cv
import numpy as np
import mediapipe as mp
from dofbot_utils.camera import create_camera
from dofbot_utils.fps import FPS


class FaceMeshDetector:
    def __init__(self, staticMode=False, maxFaces=2,
                 minDetectionCon=0.5, minTrackingCon=0.5):
        self.mpDraw = mp.solutions.drawing_utils
        self.mpFaceMesh = mp.solutions.face_mesh
        self.faceMesh = self.mpFaceMesh.FaceMesh(
            static_image_mode=staticMode,
            max_num_faces=maxFaces,
            min_detection_confidence=minDetectionCon,
            min_tracking_confidence=minTrackingCon)
        self.lmDrawSpec = mp.solutions.drawing_utils.DrawingSpec(
            color=(0, 0, 255), thickness=-1, circle_radius=3)
        self.drawSpec = self.mpDraw.DrawingSpec(
            color=(0, 255, 0), thickness=1, circle_radius=1)

    def detect(self, frame, draw=True):
        img = np.zeros(frame.shape, np.uint8)
        imgRGB = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
        results = self.faceMesh.process(imgRGB)
        if results.multi_face_landmarks:
            for faceLms in results.multi_face_landmarks:
                try:
                    conn = self.mpFaceMesh.FACEMESH_CONTOURS
                except AttributeError:
                    conn = self.mpFaceMesh.FACE_CONNECTIONS
                if draw:
                    self.mpDraw.draw_landmarks(
                        frame, faceLms, conn,
                        self.lmDrawSpec, self.drawSpec)
                self.mpDraw.draw_landmarks(
                    img, faceLms, conn,
                    self.lmDrawSpec, self.drawSpec)
        return frame, img


def frame_combine(frame, src):
    if len(frame.shape) == 3:
        frameH, frameW = frame.shape[:2]
        srcH, srcW = src.shape[:2]
        dst = np.zeros((max(frameH, srcH), frameW + srcW, 3), np.uint8)
        dst[:frameH, :frameW] = frame
        dst[:srcH, frameW:frameW + srcW] = src
    else:
        src = cv.cvtColor(src, cv.COLOR_BGR2GRAY)
        frameH, frameW = frame.shape[:2]
        srcH, srcW = src.shape[:2]
        dst = np.zeros((max(frameH, srcH), frameW + srcW), np.uint8)
        dst[:frameH, :frameW] = frame
        dst[:srcH, frameW:frameW + srcW] = src
    return dst


def main():
    detector = FaceMeshDetector()
    fps = FPS()

    cap = create_camera()
    if not cap.isOpened():
        raise RuntimeError("Failed to open camera")

    try:
        while cap.isOpened():
            ret, frame = cap.read()
            if not ret:
                continue

            frame, img = detector.detect(frame)
            fps.update_fps()
            fps.show_fps(frame)

            combined = frame_combine(frame, img)
            cv.imshow('FaceMeshDetector', combined)

            if cv.waitKey(1) & 0xFF == ord('q'):
                break
    finally:
        cap.release()
        cv.destroyAllWindows()


if __name__ == '__main__':
    main()