Welcome to DOFBOT-Ultra repository

3D Object Recognition

Please ensure that the web control program backend process is not running in the background. To close it, refer to the [Get Started Quickly - 2. Access the Terminal] section in [Disable the Auto-Start Web Control Backend Process]. After closing, please restart DofbotUltra!!!

Use MediaPipe Objectron for 3D detection and localization of objects in the camera feed. Once recognized, a 3D bounding box and coordinate axes are drawn around the object.

1、Startup

1.1、Supported Object Types

The program includes 4 object recognition models, which can be cycled through by pressing the f key at runtime:

The current model name is displayed at the top center of the screen.

1.2、Startup

• Enter the following command to start the program

xxxxxxxxxx
ros2 run dofbot_mediapipe 07_Objectron.py

After starting the program, place the corresponding object in the camera view. The image will recognize the object and draw a 3D bounding box.

• Press f key: Switch recognition model (Shoe → Chair → Cup → Camera → Shoe …)

• Press q key: Exit the program

1.2、Source Code

Raspberry Pi source path:

xxxxxxxxxx
~/dofbot_ultra_ws/src/dofbot_mediapipe/scripts/07_Objectron.py

RDK X5 source path:

xxxxxxxxxx
~/dofbot_ultra_ws/src/2d/dofbot_mediapipe/scripts/07_Objectron.py

​x
#!/usr/bin/env python3
# encoding: utf-8
import mediapipe as mp
import cv2 as cv
from dofbot_utils.camera import create_camera
from dofbot_utils.fps import FPS


class ObjectronDetector:
    def __init__(self, staticMode=False, maxObjects=5,
                 minDetectionCon=0.5, minTrackingCon=0.99):
        self.staticMode = staticMode
        self.maxObjects = maxObjects
        self.minDetectionCon = minDetectionCon
        self.minTrackingCon = minTrackingCon
        self.index = 0
        self.modelNames = ['Shoe', 'Chair', 'Cup', 'Camera']
        self.mpObjectron = mp.solutions.objectron
        self.mpDraw = mp.solutions.drawing_utils
        self._create_objectron()

    def _create_objectron(self):
        self.objectron = self.mpObjectron.Objectron(
            self.staticMode, self.maxObjects,
            self.minDetectionCon, self.minTrackingCon,
            self.modelNames[self.index])

    def detect(self, frame):
        cv.putText(frame, self.modelNames[self.index],
                   (int(frame.shape[1] / 2) - 30, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 3)
        img_RGB = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
        results = self.objectron.process(img_RGB)
        if results.detected_objects:
            for detection in results.detected_objects:
                self.mpDraw.draw_landmarks(
                    frame, detection.landmarks_2d,
                    self.mpObjectron.BOX_CONNECTIONS)
                self.mpDraw.draw_axis(frame, detection.rotation,
                                      detection.translation)
        return frame

    def switch_model(self):
        self.index = (self.index + 1) % len(self.modelNames)
        self._create_objectron()


def main():
    detector = ObjectronDetector()
    fps = FPS()

    cap = create_camera()
    if not cap.isOpened():
        raise RuntimeError("Failed to open camera")

    try:
        while cap.isOpened():
            ret, frame = cap.read()
            if not ret:
                continue

            action = cv.waitKey(1) & 0xFF
            if action == ord('q'):
                break
            if action in (ord('f'), ord('F')):
                detector.switch_model()

            frame = detector.detect(frame)
            fps.update_fps()
            fps.show_fps(frame)
            cv.imshow('Objectron', frame)
    finally:
        cap.release()
        cv.destroyAllWindows()


if __name__ == '__main__':
    main()