如何用OpenCV進行人臉疲勞檢測--基于米爾瑞芯微RK3576開發板

發布時間：2024-12-19 11:37 發布者：swiftman

關鍵詞：瑞芯微 , RK3576 , 嵌入式 , OpenCV , 米爾電子 , OpenCV

本篇源自：優秀創作者 lulugl

本文將介紹基于米爾電子MYD-LR3576開發板（米爾基于瑞芯微 RK3576開發板）的人臉疲勞檢測方案測試。

米爾基于RK3576核心板/開發板

【前言】

人臉疲勞檢測：一種通過分析人臉特征來判斷一個人是否處于疲勞狀態的技術。其原理主要基于計算機視覺和機器學習方法。當人疲勞時，面部會出現一些特征變化，如眼睛閉合程度增加、眨眼頻率變慢、打哈欠、頭部姿態改變等。
例如，通過檢測眼睛的狀態來判斷疲勞程度是一個關鍵部分。正常情況下，人的眨眼頻率相對穩定，而當疲勞時，眨眼頻率會降低，并且每次眨眼時眼睛閉合的時間可能會延長。同時，頭部可能會不自覺地下垂或者搖晃，這些特征都可以作為疲勞檢測的依據。米爾MYC-LR3576采用8核CPU+搭載6 TOPS的NPU加速器，3D GPU，能夠非常輕松的實現這個功能，下面就如何實現這一功能分享如下：

【硬件】

1、米爾MYC-LR3576開發板
2、USB攝像頭

【軟件】

1、v4l2
2、openCV
3、dlib庫：dlib 是一個現代化的 C++ 工具包，它包含了許多用于機器學習、圖像處理、數值計算等多種任務的算法和工具。它的設計目標是提供高性能、易于使用的庫，并且在開源社區中被廣泛應用。

【實現步驟】

1、安裝python-opencv
2、安裝dlib庫
3、安裝v4l2庫

【代碼實現】

1、引入cv2、dlib以及線程等：

import cv2
import dlib
import numpy as np
import time
from concurrent.futures import ThreadPoolExecutor
import threading

復制代碼

2、初始化dlib的面部檢測器和特征點預測器

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

復制代碼

3、定義計算眼睛縱橫比的函數

def eye_aspect_ratio(eye):
A = np.linalg.norm(np.array(eye[1]) - np.array(eye[5]))
B = np.linalg.norm(np.array(eye[2]) - np.array(eye[4]))
C = np.linalg.norm(np.array(eye[0]) - np.array(eye[3]))
ear = (A + B) / (2.0 * C)
return ear

復制代碼

4、定義計算頭部姿勢的函數

def get_head_pose(shape):
# 定義面部特征點的三維坐標
object_points = np.array([
(0.0, 0.0, 0.0), # 鼻尖
(0.0, -330.0, -65.0), # 下巴
(-225.0, 170.0, -135.0), # 左眼左眼角
(225.0, 170.0, -135.0), # 右眼右眼角
(-150.0, -150.0, -125.0), # 左嘴角
(150.0, -150.0, -125.0) # 右嘴角
], dtype=np.float32)
image_pts = np.float32([shape[i] for i in [30, 8, 36, 45, 48, 54]])
size = frame.shape
focal_length = size[1]
center = (size[1] // 2, size[0] // 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]], dtype="double"
)
dist_coeffs = np.zeros((4, 1))
(success, rotation_vector, translation_vector) = cv2.solvePnP(
object_points, image_pts, camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE
)
rmat, _ = cv2.Rodrigues(rotation_vector)
angles, _, _, _, _, _ = cv2.RQDecomp3x3(rmat)
return angles

復制代碼

5、定義眼睛縱橫比閾值和連續幀數閾值

EYE_AR_THRESH = 0.3
EYE_AR_CONSEC_FRAMES = 48

復制代碼

6、打開攝像頭
我們先使用v4l2-ctl --list-devices來例出接在開發板上的列表信息：

USB Camera: USB Camera (usb-xhci-hcd.0.auto-1.2):
/dev/video60
/dev/video61
/dev/media7

復制代碼

在代碼中填入60為攝像頭的編號：

cap = cv2.VideoCapture(60)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 480) # 降低分辨率
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 320)

復制代碼

7、創建多線程處理函數，實現采集與分析分離：

# 多線程處理函數
def process_frame(frame):
global COUNTER, TOTAL
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = detector(gray, 0) # 第二個參數為0，表示不使用upsampling
for face in faces:
landmarks = predictor(gray, face)
shape = [(landmarks.part(i).x, landmarks.part(i).y) for i in range(68)]
left_eye = shape[36:42]
right_eye = shape[42:48]
left_ear = eye_aspect_ratio(left_eye)
right_ear = eye_aspect_ratio(right_eye)
ear = (left_ear + right_ear) / 2.0
if ear < EYE_AR_THRESH:
with lock:
COUNTER += 1
else:
with lock:
if COUNTER >= EYE_AR_CONSEC_FRAMES:
TOTAL += 1
COUNTER = 0
# 繪制68個特征點
for n in range(0, 68):
x, y = shape[n]
cv2.circle(frame, (x, y), 2, (0, 255, 0), -1)
cv2.putText(frame, f"Eye AR: {ear:.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Blink Count: {TOTAL}", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
# 計算頭部姿勢
angles = get_head_pose(shape)
pitch, yaw, roll = angles
cv2.putText(frame, f"Pitch: {pitch:.2f}", (10, 120), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Yaw: {yaw:.2f}", (10, 150), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Roll: {roll:.2f}", (10, 180), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
# 判斷疲勞狀態
if COUNTER >= EYE_AR_CONSEC_FRAMES or abs(pitch) > 30 or abs(yaw) > 30 or abs(roll) > 30:
cv2.putText(frame, "Fatigue Detected!", (10, 210), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
return frame

復制代碼

8、創建圖像顯示線程：

with ThreadPoolExecutor(max_workers=2) as executor:
future_to_frame = {}
while True:
ret, frame = cap.read()
if not ret:
break
# 提交當前幀到線程池
future = executor.submit(process_frame, frame.copy())
future_to_frame[future] = frame
# 獲取已完成的任務結果
for future in list(future_to_frame.keys()):
if future.done():
processed_frame = future.result()
cv2.imshow("Frame", processed_frame)
del future_to_frame[future]
break
# 計算幀數
fps_counter += 1
elapsed_time = time.time() - start_time
if elapsed_time > 1.0:
fps = fps_counter / elapsed_time
fps_counter = 0
start_time = time.time()
cv2.putText(processed_frame, f"FPS: {fps:.2f}", (10, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
if cv2.waitKey(1) & 0xFF == ord('q'):

復制代碼

實現效果：

根據檢測的結果，我們就可以來實現疲勞提醒等等的功能。
整體代碼如下：

import cv2
import dlib
import numpy as np
import time
from concurrent.futures import ThreadPoolExecutor
import threading
# 初始化dlib的面部檢測器和特征點預測器
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
# 修改字體大小
font_scale = 0.5 # 原來的字體大小是0.7，現在改為0.5
# 定義計算眼睛縱橫比的函數
def eye_aspect_ratio(eye):
A = np.linalg.norm(np.array(eye[1]) - np.array(eye[5]))
B = np.linalg.norm(np.array(eye[2]) - np.array(eye[4]))
C = np.linalg.norm(np.array(eye[0]) - np.array(eye[3]))
ear = (A + B) / (2.0 * C)
return ear
# 定義計算頭部姿勢的函數
def get_head_pose(shape):
# 定義面部特征點的三維坐標
object_points = np.array([
(0.0, 0.0, 0.0), # 鼻尖
(0.0, -330.0, -65.0), # 下巴
(-225.0, 170.0, -135.0), # 左眼左眼角
(225.0, 170.0, -135.0), # 右眼右眼角
(-150.0, -150.0, -125.0), # 左嘴角
(150.0, -150.0, -125.0) # 右嘴角
], dtype=np.float32)
image_pts = np.float32([shape[i] for i in [30, 8, 36, 45, 48, 54]])
size = frame.shape
focal_length = size[1]
center = (size[1] // 2, size[0] // 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]], dtype="double"
)
dist_coeffs = np.zeros((4, 1))
(success, rotation_vector, translation_vector) = cv2.solvePnP(
object_points, image_pts, camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE
)
rmat, _ = cv2.Rodrigues(rotation_vector)
angles, _, _, _, _, _ = cv2.RQDecomp3x3(rmat)
return angles
# 定義眼睛縱橫比閾值和連續幀數閾值
EYE_AR_THRESH = 0.3
EYE_AR_CONSEC_FRAMES = 48
# 初始化計數器
COUNTER = 0
TOTAL = 0
# 創建鎖對象
lock = threading.Lock()
# 打開攝像頭
cap = cv2.VideoCapture(60)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 480) # 降低分辨率
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 320)
# 初始化幀計數器和時間戳
fps_counter = 0
start_time = time.time()
# 多線程處理函數
def process_frame(frame):
global COUNTER, TOTAL
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = detector(gray, 0) # 第二個參數為0，表示不使用upsampling
for face in faces:
landmarks = predictor(gray, face)
shape = [(landmarks.part(i).x, landmarks.part(i).y) for i in range(68)]
left_eye = shape[36:42]
right_eye = shape[42:48]
left_ear = eye_aspect_ratio(left_eye)
right_ear = eye_aspect_ratio(right_eye)
ear = (left_ear + right_ear) / 2.0
if ear < EYE_AR_THRESH:
with lock:
COUNTER += 1
else:
with lock:
if COUNTER >= EYE_AR_CONSEC_FRAMES:
TOTAL += 1
COUNTER = 0
# 繪制68個特征點
for n in range(0, 68):
x, y = shape[n]
cv2.circle(frame, (x, y), 2, (0, 255, 0), -1)
cv2.putText(frame, f"Eye AR: {ear:.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Blink Count: {TOTAL}", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
# 計算頭部姿勢
angles = get_head_pose(shape)
pitch, yaw, roll = angles
cv2.putText(frame, f"Pitch: {pitch:.2f}", (10, 120), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Yaw: {yaw:.2f}", (10, 150), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
cv2.putText(frame, f"Roll: {roll:.2f}", (10, 180), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
# 判斷疲勞狀態
if COUNTER >= EYE_AR_CONSEC_FRAMES or abs(pitch) > 30 or abs(yaw) > 30 or abs(roll) > 30:
cv2.putText(frame, "Fatigue Detected!", (10, 210), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 255), 2)
return frame
with ThreadPoolExecutor(max_workers=2) as executor:
future_to_frame = {}
while True:
ret, frame = cap.read()
if not ret:
break
# 提交當前幀到線程池
future = executor.submit(process_frame, frame.copy())
future_to_frame[future] = frame
# 獲取已完成的任務結果
for future in list(future_to_frame.keys()):
if future.done():
processed_frame = future.result()
cv2.imshow("Frame", processed_frame)
del future_to_frame[future]
break
# 計算幀數
fps_counter += 1
elapsed_time = time.time() - start_time
if elapsed_time > 1.0:
fps = fps_counter / elapsed_time
fps_counter = 0
start_time = time.time()
cv2.putText(processed_frame, f"FPS: {fps:.2f}", (10, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# 釋放攝像頭并關閉所有窗口
cap.release()
cv2.destroyAllWindows()