Welcome to Muto RS Robot repository
6)Face effects
6.Face effects
6.1. Introduction
MediaPipe is an open source data stream processing machine learning application development framework developed by Google. It is a graph-based data processing pipeline for building and using multiple forms of data sources, such as video, audio, sensor data, and any time series data. MediaPipe is cross-platform and can run on embedded platforms (Raspberry Pi, etc.), mobile devices (iOS and Android), workstations and servers, and supports mobile GPU acceleration. MediaPipe provides cross-platform, customizable ML solutions for real-time and streaming media. The core framework of MediaPipe is implemented in C++ and provides support for languages such as Java and Objective C. The main concepts of MediaPipe include Packet, Stream, Calculator, Graph and Subgraph.
MediaPipe Features:
- End-to-end acceleration: Built-in fast ML inference and processing accelerates even on commodity hardware.
- Build once, deploy anywhere: Unified solution for Android, iOS, desktop/cloud, web and IoT.
- Ready-to-use solutions: cutting-edge ML solutions that showcase the full capabilities of the framework.
- Free and open source: frameworks and solutions under Apache2.0, fully extensible and customizable.
6.2. Face special effects
6.2.1.Startup
Note: Before running this case, please make sure that the [/dev/video0] device has been successfully mounted into the docker container, otherwise the camera will not be opened.
Enter the docker container and execute:
cd /root/yahboomcar_ros2_ws/yahboomcar_ws/src/yahboomcar_mediapipe/yahboomcar_mediapipe#Face special effectspython3 06_FaceLandmarks.py 
6.2.2.Source code
Source code location:
xxxxxxxxxx/root/yahboomcar_ros2_ws/yahboomcar_ws/src/yahboomcar_mediapipe/yahboomcar_mediapipe/06_FaceLandmarks.py
x#!/usr/bin/env python3# encoding: utf-8import timeimport dlibimport cv2 as cvimport numpy as np
class FaceLandmarks: def __init__(self, dat_file): self.hog_face_detector = dlib.get_frontal_face_detector() self.dlib_facelandmark = dlib.shape_predictor(dat_file)
def get_face(self, frame, draw=True): gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) self.faces = self.hog_face_detector(gray) for face in self.faces: self.face_landmarks = self.dlib_facelandmark(gray, face) if draw: for n in range(68): x = self.face_landmarks.part(n).x y = self.face_landmarks.part(n).y cv.circle(frame, (x, y), 2, (0, 255, 255), 2) cv.putText(frame, str(n), (x, y), cv.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2) return frame
def get_lmList(self, frame, p1, p2, draw=True): lmList = [] if len(self.faces) != 0: for n in range(p1, p2): x = self.face_landmarks.part(n).x y = self.face_landmarks.part(n).y lmList.append([x, y]) if draw: next_point = n + 1 if n == p2 - 1: next_point = p1 x2 = self.face_landmarks.part(next_point).x y2 = self.face_landmarks.part(next_point).y cv.line(frame, (x, y), (x2, y2), (0, 255, 0), 1) return lmList
def get_lipList(self, frame, lipIndexlist, draw=True): lmList = [] if len(self.faces) != 0: for n in range(len(lipIndexlist)): x = self.face_landmarks.part(lipIndexlist[n]).x y = self.face_landmarks.part(lipIndexlist[n]).y lmList.append([x, y]) if draw: next_point = n + 1 if n == len(lipIndexlist) - 1: next_point = 0 x2 = self.face_landmarks.part(lipIndexlist[next_point]).x y2 = self.face_landmarks.part(lipIndexlist[next_point]).y cv.line(frame, (x, y), (x2, y2), (0, 255, 0), 1) return lmList
def prettify_face(self, frame, eye=True, lips=True, eyebrow=True, draw=True): if eye: leftEye = landmarks.get_lmList(frame, 36, 42) rightEye = landmarks.get_lmList(frame, 42, 48) if draw: if len(leftEye) != 0: frame = cv.fillConvexPoly(frame, np.mat(leftEye), (0, 0, 0)) if len(rightEye) != 0: frame = cv.fillConvexPoly(frame, np.mat(rightEye), (0, 0, 0)) if lips: lipIndexlistA = [51, 52, 53, 54, 64, 63, 62] lipIndexlistB = [48, 49, 50, 51, 62, 61, 60] lipsUpA = landmarks.get_lipList(frame, lipIndexlistA, draw=True) lipsUpB = landmarks.get_lipList(frame, lipIndexlistB, draw=True) lipIndexlistA = [57, 58, 59, 48, 67, 66] lipIndexlistB = [54, 55, 56, 57, 66, 65, 64] lipsDownA = landmarks.get_lipList(frame, lipIndexlistA, draw=True) lipsDownB = landmarks.get_lipList(frame, lipIndexlistB, draw=True) if draw: if len(lipsUpA) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsUpA), (249, 0, 226)) if len(lipsUpB) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsUpB), (249, 0, 226)) if len(lipsDownA) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsDownA), (249, 0, 226)) if len(lipsDownB) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsDownB), (249, 0, 226)) if eyebrow: lefteyebrow = landmarks.get_lmList(frame, 17, 22) righteyebrow = landmarks.get_lmList(frame, 22, 27) if draw: if len(lefteyebrow) != 0: frame = cv.fillConvexPoly(frame, np.mat(lefteyebrow), (255, 255, 255)) if len(righteyebrow) != 0: frame = cv.fillConvexPoly(frame, np.mat(righteyebrow), (255, 255, 255)) return frame
if __name__ == '__main__': capture = cv.VideoCapture(0) capture.set(6, cv.VideoWriter.fourcc('M', 'J', 'P', 'G')) capture.set(cv.CAP_PROP_FRAME_WIDTH, 640) capture.set(cv.CAP_PROP_FRAME_HEIGHT, 480) print("capture get FPS : ", capture.get(cv.CAP_PROP_FPS)) pTime, cTime = 0, 0 dat_file = "./file/shape_predictor_68_face_landmarks.dat" landmarks = FaceLandmarks(dat_file) while capture.isOpened(): ret, frame = capture.read() # frame = cv.flip(frame, 1) frame = landmarks.get_face(frame, draw=False) frame = landmarks.prettify_face(frame, eye=True, lips=True, eyebrow=True, draw=True) if cv.waitKey(1) & 0xFF == ord('q'): break cTime = time.time() fps = 1 / (cTime - pTime) pTime = cTime text = "FPS : " + str(int(fps)) cv.putText(frame, text, (20, 30), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1) cv.imshow('frame', frame) capture.release() cv.destroyAllWindows()