ZF/insightface
1
0
Fork 0
mirror of https://github.com/deepinsight/insightface.git synced 2026-08-07 02:27:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update InspireFace to 1.1.4

Browse Source
This commit is contained in:
JingyuYan
2024-07-05 21:54:55 +08:00
parent a1b9206a64
commit dfc72baf25
82 changed files with 1448 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cpp-package/inspireface/cpp/inspireface/CMakeLists.txt
View File

@@ -55,6 +55,7 @@ set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_ar
link_directories(${MNN_LIBS})
if(ISF_BUILD_SHARED_LIBS)
add_definitions("-DISF_BUILD_SHARED_LIBS")
add_library(InspireFace SHARED ${SOURCE_FILES})
else()
add_library(InspireFace STATIC ${SOURCE_FILES})

83
cpp-package/inspireface/cpp/inspireface/c_api/inspireface.cc
View File

@@ -100,13 +100,13 @@ HResult HFReleaseInspireFaceSession(HFSession handle) {
HResult HFCreateInspireFaceSession(HFSessionCustomParameter parameter, HFDetectMode detectMode, HInt32 maxDetectFaceNum, HInt32 detectPixelLevel, HInt32 trackByDetectModeFPS, HFSession *handle) {
inspire::ContextCustomParameter param;
param.enable_mask_detect = parameter.enable_mask_detect;
param.enable_age = parameter.enable_age;
param.enable_face_attribute = parameter.enable_face_quality;
param.enable_liveness = parameter.enable_liveness;
param.enable_face_quality = parameter.enable_face_quality;
param.enable_gender = parameter.enable_gender;
param.enable_interaction_liveness = parameter.enable_interaction_liveness;
param.enable_ir_liveness = parameter.enable_ir_liveness;
param.enable_recognition = parameter.enable_recognition;
param.enable_face_attribute = parameter.enable_face_attribute;
inspire::DetectMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
if (detectMode == HF_DETECT_MODE_LIGHT_TRACK) {
detMode = inspire::DETECT_MODE_LIGHT_TRACK;
@@ -138,11 +138,8 @@ HResult HFCreateInspireFaceSessionOptional(HOption customOption, HFDetectMode de
if (customOption & HF_ENABLE_IR_LIVENESS) {
param.enable_ir_liveness = true;
}
if (customOption & HF_ENABLE_AGE_PREDICT) {
param.enable_age = true;
}
if (customOption & HF_ENABLE_GENDER_PREDICT) {
param.enable_gender = true;
if (customOption & HF_ENABLE_FACE_ATTRIBUTE) {
param.enable_face_attribute = true;
}
if (customOption & HF_ENABLE_MASK_DETECT) {
param.enable_mask_detect = true;
@@ -281,6 +278,33 @@ HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize) {
return HSUCCEED;
}
HResult HFGetNumOfFaceDenseLandmark(HPInt32 num) {
*num = 106;
return HSUCCEED;
}
HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num) {
if (num != 106) {
return HERR_SESS_LANDMARK_NUM_NOT_MATCH;
}
inspire::FaceBasicData data;
data.dataSize = singleFace.size;
data.data = singleFace.data;
HyperFaceData face = {0};
HInt32 ret;
ret = DeserializeHyperFaceData((char* )data.data, data.dataSize, face);
if (ret != HSUCCEED) {
return ret;
}
for (size_t i = 0; i < num; i++)
{
landmarks[i].x = face.densityLandmark[i].x;
landmarks[i].y = face.densityLandmark[i].y;
}
return HSUCCEED;
}
HResult HFFeatureHubFaceSearchThresholdSetting(float threshold) {
FEATURE_HUB->SetRecognitionThreshold(threshold);
return HSUCCEED;
@@ -481,13 +505,13 @@ HResult HFMultipleFacePipelineProcess(HFSession session, HFImageStream streamHan
}
inspire::ContextCustomParameter param;
param.enable_mask_detect = parameter.enable_mask_detect;
param.enable_age = parameter.enable_age;
param.enable_face_attribute = parameter.enable_face_quality;
param.enable_liveness = parameter.enable_liveness;
param.enable_face_quality = parameter.enable_face_quality;
param.enable_gender = parameter.enable_gender;
param.enable_interaction_liveness = parameter.enable_interaction_liveness;
param.enable_ir_liveness = parameter.enable_ir_liveness;
param.enable_recognition = parameter.enable_recognition;
param.enable_face_attribute = parameter.enable_face_attribute;
HResult ret;
std::vector<inspire::HyperFaceData> data;
@@ -535,11 +559,8 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
if (customOption & HF_ENABLE_IR_LIVENESS) {
param.enable_ir_liveness = true;
}
if (customOption & HF_ENABLE_AGE_PREDICT) {
param.enable_age = true;
}
if (customOption & HF_ENABLE_GENDER_PREDICT) {
param.enable_gender = true;
if (customOption & HF_ENABLE_FACE_ATTRIBUTE) {
param.enable_face_attribute = true;
}
if (customOption & HF_ENABLE_MASK_DETECT) {
param.enable_mask_detect = true;
@@ -549,7 +570,7 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
}
if (customOption & HF_ENABLE_INTERACTION) {
param.enable_interaction_liveness = true;
}
}
HResult ret;
@@ -633,6 +654,38 @@ HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFlo
}
HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
result->num = ctx->impl.GetFaceInteractionLeftEyeStatusCache().size();
result->leftEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionLeftEyeStatusCache().data();
result->rightEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionRightEyeStatusCache().data();
return HSUCCEED;
}
HResult HFGetFaceAttributeResult(HFSession session, PHFFaceAttributeResult results) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
results->num = ctx->impl.GetFaceAgeBracketResultsCache().size();
results->race = (HPInt32 )ctx->impl.GetFaceRaceResultsCache().data();
results->gender = (HPInt32 )ctx->impl.GetFaceGenderResultsCache().data();
results->ageBracket = (HPInt32 )ctx->impl.GetFaceAgeBracketResultsCache().data();
return HSUCCEED;
}
HResult HFFeatureHubGetFaceCount(HInt32* count) {
*count = FEATURE_HUB->GetFaceFeatureCount();
return HSUCCEED;

81
cpp-package/inspireface/cpp/inspireface/c_api/inspireface.h
View File

@@ -10,7 +10,7 @@
#include "herror.h"
#if defined(_WIN32)
#ifdef HYPER_BUILD_SHARED_LIB
#ifdef ISF_BUILD_SHARED_LIBS
#define HYPER_CAPI_EXPORT __declspec(dllexport)
#else
#define HYPER_CAPI_EXPORT
@@ -29,8 +29,8 @@ extern "C" {
#define HF_ENABLE_LIVENESS 0x00000004 ///< Flag to enable RGB liveness detection feature.
#define HF_ENABLE_IR_LIVENESS 0x00000008 ///< Flag to enable IR (Infrared) liveness detection feature.
#define HF_ENABLE_MASK_DETECT 0x00000010 ///< Flag to enable mask detection feature.
#define HF_ENABLE_AGE_PREDICT 0x00000020 ///< Flag to enable age prediction feature.
#define HF_ENABLE_GENDER_PREDICT 0x00000040 ///< Flag to enable gender prediction feature.
#define HF_ENABLE_FACE_ATTRIBUTE 0x00000020 ///< Flag to enable face attribute prediction feature.
#define HF_ENABLE_PLACEHOLDER_ 0x00000040 ///< -
#define HF_ENABLE_QUALITY 0x00000080 ///< Flag to enable face quality assessment feature.
#define HF_ENABLE_INTERACTION 0x00000100 ///< Flag to enable interaction feature.
@@ -125,9 +125,8 @@ typedef struct HFSessionCustomParameter {
HInt32 enable_liveness; ///< Enable RGB liveness detection feature.
HInt32 enable_ir_liveness; ///< Enable IR liveness detection feature.
HInt32 enable_mask_detect; ///< Enable mask detection feature.
HInt32 enable_age; ///< Enable age prediction feature.
HInt32 enable_gender; ///< Enable gender prediction feature.
HInt32 enable_face_quality; ///< Enable face quality detection feature.
HInt32 enable_face_attribute; ///< Enable face attribute prediction feature.
HInt32 enable_interaction_liveness; ///< Enable interaction for liveness detection feature.
} HFSessionCustomParameter, *PHFSessionCustomParameter;
@@ -149,7 +148,7 @@ typedef enum HFDetectMode {
* @param detectMode Detection mode to be used.
* @param maxDetectFaceNum Maximum number of faces to detect.
* @param detectPixelLevel Modify the input resolution level of the detector, the larger the better,
* the need to input a multiple of 160, such as 160, 320, 640, the default value -1 is 160.
* the need to input a multiple of 160, such as 160, 320, 640, the default value -1 is 320.
* @param trackByDetectModeFPS If you are using the MODE_TRACK_BY_DETECTION tracking mode,
* this value is used to set the fps frame rate of your current incoming video stream, which defaults to -1 at 30fps.
* @param handle Pointer to the context handle that will be returned.
@@ -298,6 +297,23 @@ HYPER_CAPI_EXPORT extern HResult HFCopyFaceBasicToken(HFFaceBasicToken token, HP
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize);
/**
* @brief Retrieve the number of dense facial landmarks.
* @param num Number of dense facial landmarks
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetNumOfFaceDenseLandmark(HPInt32 num);
/**
* @brief When you pass in a valid facial token, you can retrieve a set of dense facial landmarks.
* The memory for the dense landmarks must be allocated by you.
* @param singleFace Basic token representing a single face.
* @param landmarks Pre-allocated memory address of the array for 2D floating-point coordinates.
* @param num Number of landmark points
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num);
/************************************************************************
* Face Recognition
************************************************************************/
@@ -618,6 +634,59 @@ HYPER_CAPI_EXPORT extern HResult HFGetFaceQualityConfidence(HFSession session, P
*/
HYPER_CAPI_EXPORT extern HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFloat *confidence);
/**
* @brief Some facial states in the face interaction module.
*/
typedef struct HFFaceIntereactionResult {
HInt32 num; ///< Number of faces detected.
HPFloat leftEyeStatusConfidence; ///< Left eye state: confidence close to 1 means open, close to 0 means closed.
HPFloat rightEyeStatusConfidence; ///< Right eye state: confidence close to 1 means open, close to 0 means closed.
} HFFaceIntereactionResult, *PHFFaceIntereactionResult;
HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result);
/**
* @brief Struct representing face attribute results.
*
* This struct holds the race, gender, and age bracket attributes for a detected face.
*/
typedef struct HFFaceAttributeResult {
HInt32 num; ///< Number of faces detected.
HPInt32 race; ///< Race of the detected face.
///< 0: Black;
///< 1: Asian;
///< 2: Latino/Hispanic;
///< 3: Middle Eastern;
///< 4: White;
HPInt32 gender; ///< Gender of the detected face.
///< 0: Female;
///< 1: Male;
HPInt32 ageBracket; ///< Age bracket of the detected face.
///< 0: 0-2 years old;
///< 1: 3-9 years old;
///< 2: 10-19 years old;
///< 3: 20-29 years old;
///< 4: 30-39 years old;
///< 5: 40-49 years old;
///< 6: 50-59 years old;
///< 7: 60-69 years old;
///< 8: more than 70 years old;
} HFFaceAttributeResult, *PHFFaceAttributeResult;
/**
* @brief Get the face attribute results.
*
* This function retrieves the attribute results such as race, gender, and age bracket
* for faces detected in the current context.
*
* @param session Handle to the session.
* @param results Pointer to the structure where face attribute results will be stored.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceAttributeResult(HFSession session, PHFFaceAttributeResult results);
/************************************************************************
* System Function
************************************************************************/

4
cpp-package/inspireface/cpp/inspireface/c_api/intypedef.h
View File

@@ -31,5 +31,9 @@ typedef struct HFaceRect {
HInt32 height; ///< Height of the rectangle.
} HFaceRect; ///< Rectangle representing a face region.
typedef struct HPoint2f{
HFloat x; ///< X-coordinate
HFloat y; ///< Y-coordinate
} HPoint2f;
#endif //HYPERFACEREPO_INTYPEDEF_H

9
cpp-package/inspireface/cpp/inspireface/common/face_data/data_tools.h
View File

@@ -96,6 +96,15 @@ inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj
data.face3DAngle.pitch = obj.high_result.pitch;
data.face3DAngle.roll = obj.high_result.roll;
data.face3DAngle.yaw = obj.high_result.yaw;
const auto &lmk = obj.landmark_smooth_aux_.back();
for (size_t i = 0; i < lmk.size(); i++)
{
data.densityLandmark[i].x = lmk[i].x;
data.densityLandmark[i].y = lmk[i].y;
}
return data;
}

19
cpp-package/inspireface/cpp/inspireface/common/face_data/face_data_type.h
View File

@@ -57,15 +57,16 @@ typedef struct TransMatrix {
* Struct to represent hyper face data.
*/
typedef struct HyperFaceData {
int trackState; ///< Track state
int inGroupIndex; ///< Index within a group
int trackId; ///< Track ID
int trackCount; ///< Track count
FaceRect rect; ///< Face rectangle
TransMatrix trans; ///< Transformation matrix
Point2F keyPoints[5]; ///< Key points (e.g., landmarks)
Face3DAngle face3DAngle; ///< 3D face angles
float quality[5]; ///< Quality values for key points
int trackState; ///< Track state
int inGroupIndex; ///< Index within a group
int trackId; ///< Track ID
int trackCount; ///< Track count
FaceRect rect; ///< Face rectangle
TransMatrix trans; ///< Transformation matrix
Point2F keyPoints[5]; ///< Key points (e.g., landmarks)
Face3DAngle face3DAngle; ///< 3D face angles
float quality[5]; ///< Quality values for key points
Point2F densityLandmark[106]; ///< Face density landmark
} HyperFaceData;
} // namespace inspire

4
cpp-package/inspireface/cpp/inspireface/common/face_info/face_object.h
View File

@@ -312,6 +312,10 @@ public:
face_id_ = id;
}
std::vector<float> left_eye_status_;
std::vector<float> right_eye_status_;
private:
TRACK_STATE tracking_state_;
// std::shared_ptr<FaceAction> face_action_;

73
cpp-package/inspireface/cpp/inspireface/face_context.cpp
View File

@@ -42,8 +42,7 @@ int32_t FaceContext::Configuration(DetectMode detect_mode,
INSPIRE_LAUNCH->getMArchive(),
param.enable_liveness,
param.enable_mask_detect,
param.enable_age,
param.enable_gender,
param.enable_face_attribute,
param.enable_interaction_liveness
);
@@ -62,6 +61,11 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream &image) {
m_yaw_results_cache_.clear();
m_pitch_results_cache_.clear();
m_quality_score_results_cache_.clear();
m_react_left_eye_results_cache_.clear();
m_react_right_eye_results_cache_.clear();
m_quality_score_results_cache_.clear();
m_attribute_race_results_cache_.clear();
m_attribute_gender_results_cache_.clear();
if (m_face_track_ == nullptr) {
return HERR_SESS_TRACKER_FAILURE;
}
@@ -129,6 +133,11 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
std::lock_guard<std::mutex> lock(m_mtx_);
m_mask_results_cache_.resize(faces.size(), -1.0f);
m_rgb_liveness_results_cache_.resize(faces.size(), -1.0f);
m_react_left_eye_results_cache_.resize(faces.size(), -1.0f);
m_react_right_eye_results_cache_.resize(faces.size(), -1.0f);
m_attribute_race_results_cache_.resize(faces.size(), -1);
m_attribute_gender_results_cache_.resize(faces.size(), -1);
m_attribute_age_results_cache_.resize(faces.size(), -1);
for (int i = 0; i < faces.size(); ++i) {
const auto &face = faces[i];
// RGB Liveness Detect
@@ -147,19 +156,48 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
}
m_mask_results_cache_[i] = m_face_pipeline_->faceMaskCache;
}
// Age prediction
if (param.enable_age) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_AGE);
// Face attribute prediction
if (param.enable_face_attribute) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_ATTRIBUTE);
if (ret != HSUCCEED) {
return ret;
}
m_attribute_race_results_cache_[i] = m_face_pipeline_->faceAttributeCache[0];
m_attribute_gender_results_cache_[i] = m_face_pipeline_->faceAttributeCache[1];
m_attribute_age_results_cache_[i] = m_face_pipeline_->faceAttributeCache[2];
}
// Gender prediction
if (param.enable_age) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_GENDER);
// Face interaction
if (param.enable_interaction_liveness) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_INTERACTION);
if (ret != HSUCCEED) {
return ret;
}
// Get eyes status
m_react_left_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[0];
m_react_right_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[1];
// Special handling: ff it is a tracking state, it needs to be filtered
if (face.trackState > 0)
{
auto idx = face.inGroupIndex;
if (idx < m_face_track_->trackingFace.size()) {
auto& target = m_face_track_->trackingFace[idx];
if (target.GetTrackingId() == face.trackId) {
auto new_eye_left = EmaFilter(m_face_pipeline_->eyesStatusCache[0], target.left_eye_status_, 8, 0.2f);
auto new_eye_right = EmaFilter(m_face_pipeline_->eyesStatusCache[1], target.right_eye_status_, 8, 0.2f);
if (face.trackState > 1) {
// The filtered value can be obtained only in the tracking state
m_react_left_eye_results_cache_[i] = new_eye_left;
m_react_right_eye_results_cache_[i] = new_eye_right;
}
} else {
INSPIRE_LOGD("Serialized objects cannot connect to trace objects in memory, and there may be some problems");
}
} else {
INSPIRE_LOGW("The index of the trace object does not match the trace list in memory, and there may be some problems");
}
}
}
}
@@ -212,11 +250,30 @@ const std::vector<float>& FaceContext::GetFaceQualityScoresResultsCache() const
return m_quality_score_results_cache_;
}
const std::vector<float>& FaceContext::GetFaceInteractionLeftEyeStatusCache() const {
return m_react_left_eye_results_cache_;
}
const std::vector<float>& FaceContext::GetFaceInteractionRightEyeStatusCache() const {
return m_react_right_eye_results_cache_;
}
const Embedded& FaceContext::GetFaceFeatureCache() const {
return m_face_feature_cache_;
}
const std::vector<int>& FaceContext::GetFaceRaceResultsCache() const {
return m_attribute_race_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceGenderResultsCache() const {
return m_attribute_gender_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceAgeBracketResultsCache() const {
return m_attribute_age_results_cache_;
}
int32_t FaceContext::FaceFeatureExtract(CameraStream &image, FaceBasicData& data) {
std::lock_guard<std::mutex> lock(m_mtx_);
int32_t ret;

38
cpp-package/inspireface/cpp/inspireface/face_context.h
View File

@@ -37,8 +37,7 @@ typedef struct CustomPipelineParameter {
bool enable_liveness = false; ///< Enable RGB liveness detection feature
bool enable_ir_liveness = false; ///< Enable IR (Infrared) liveness detection feature
bool enable_mask_detect = false; ///< Enable mask detection feature
bool enable_age = false; ///< Enable age prediction feature
bool enable_gender = false; ///< Enable gender prediction feature
bool enable_face_attribute = false; ///< Enable face attribute prediction feature
bool enable_face_quality = false; ///< Enable face quality assessment feature
bool enable_interaction_liveness = false; ///< Enable interactive liveness detection feature
@@ -232,6 +231,36 @@ public:
*/
const std::vector<float>& GetFaceQualityScoresResultsCache() const;
/**
* @brief Gets the cache of left eye status predict results.
* @return A const reference to a vector containing eye status predict results.
*/
const std::vector<float>& GetFaceInteractionLeftEyeStatusCache() const;
/**
* @brief Gets the cache of right eye status predict results.
* @return A const reference to a vector containing eye status predict results.
*/
const std::vector<float>& GetFaceInteractionRightEyeStatusCache() const;
/**
* @brief Gets the cache of face attribute rece results.
* @return A const reference to a vector containing face attribute rece results.
*/
const std::vector<int>& GetFaceRaceResultsCache() const;
/**
* @brief Gets the cache of face attribute gender results.
* @return A const reference to a vector containing face attribute gender results.
*/
const std::vector<int>& GetFaceGenderResultsCache() const;
/**
* @brief Gets the cache of face attribute age bracket results.
* @return A const reference to a vector containing face attribute age bracket results.
*/
const std::vector<int>& GetFaceAgeBracketResultsCache() const;
/**
* @brief Gets the cache of the current face features.
* @return A const reference to the Embedded object containing current face feature data.
@@ -263,6 +292,11 @@ private:
std::vector<float> m_mask_results_cache_; ///< Cache for mask detection results
std::vector<float> m_rgb_liveness_results_cache_; ///< Cache for RGB liveness detection results
std::vector<float> m_quality_score_results_cache_; ///< Cache for RGB face quality score results
std::vector<float> m_react_left_eye_results_cache_; ///< Cache for Left eye state in face interaction
std::vector<float> m_react_right_eye_results_cache_; ///< Cache for Right eye state in face interaction
std::vector<int> m_attribute_race_results_cache_;
std::vector<int> m_attribute_gender_results_cache_;
std::vector<int> m_attribute_age_results_cache_;
Embedded m_face_feature_cache_; ///< Cache for current face feature data
std::mutex m_mtx_; ///< Mutex for thread safety.

1
cpp-package/inspireface/cpp/inspireface/herror.h
View File

@@ -27,6 +27,7 @@
#define HERR_SESS_TRACKER_FAILURE (HERR_SESS_BASE+3) // Tracker module not initialized
#define HERR_SESS_INVALID_RESOURCE (HERR_SESS_BASE+10) // Invalid static resource
#define HERR_SESS_NUM_OF_MODELS_NOT_MATCH (HERR_SESS_BASE+11) // Number of models does not match
#define HERR_SESS_LANDMARK_NUM_NOT_MATCH (HERR_SESS_BASE+20) // The number of input landmark points does not match
#define HERR_SESS_PIPELINE_FAILURE (HERR_SESS_BASE+8) // Pipeline module not initialized

2
cpp-package/inspireface/cpp/inspireface/information.h
View File

@@ -7,6 +7,6 @@
#define INSPIRE_FACE_VERSION_MAJOR_STR "1"
#define INSPIRE_FACE_VERSION_MINOR_STR "1"
#define INSPIRE_FACE_VERSION_PATCH_STR "2"
#define INSPIRE_FACE_VERSION_PATCH_STR "4"
#endif //HYPERFACEREPO_INFORMATION_H

9
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_mnn.cpp
View File

@@ -249,6 +249,15 @@ int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input
/* Convert color type */
// LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
// LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
// !!!!!! BUG !!!!!!!!!
// When initializing, setting the image channel to 3 and the tensor channel to 1,
// and configuring the processing to convert the color image to grayscale may cause some bugs.
// For example, the image channel might automatically change to 1.
// This issue has not been fully investigated,
// so it's necessary to manually convert the image to grayscale before input.
// !!!!!! BUG !!!!!!!!!
if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
if (input_tensor_info.image_info.swap_color) {

54
cpp-package/inspireface/cpp/inspireface/middleware/utils.h
View File

@@ -481,11 +481,9 @@ inline cv::Mat ScaleAffineMatrix(const cv::Mat &affine, float scale,
return m;
}
template<typename T>
inline int ArgMax(const std::vector<T> data, int start, int end) {
int diff = std::max_element(data.begin() + start, data.begin() + end) -
(data.begin() + start);
return diff;
template<class ForwardIterator>
inline size_t argmax(ForwardIterator first, ForwardIterator last) {
return std::distance(first, std::max_element(first, last));
}
inline void RotPoints(std::vector<cv::Point2f> &pts, float angle) {
@@ -650,6 +648,52 @@ inline bool isShortestSideGreaterThan(const cv::Rect_<T>& rect, T value, float s
return shortestSide > value;
}
/**
* @brief Computes the affine transformation matrix for face cropping.
* @param rect Rectangle representing the face in the image.
* @return cv::Mat The computed affine transformation matrix.
*/
inline cv::Mat ComputeCropMatrix(const cv::Rect2f &rect, int width, int height) {
float x = rect.x;
float y = rect.y;
float w = rect.width;
float h = rect.height;
float cx = x + w / 2;
float cy = y + h / 2;
float length = std::max(w, h) * 1.5 / 2;
float x1 = cx - length;
float y1 = cy - length;
float x2 = cx + length;
float y2 = cy + length;
cv::Rect2f padding_rect(x1, y1, x2 - x1, y2 - y1);
std::vector<cv::Point2f> rect_pts = Rect2Points(padding_rect);
rect_pts.erase(rect_pts.end() - 1);
std::vector<cv::Point2f> dst_pts = {{0, 0}, {(float )width, 0}, {(float )width, (float )height}};
cv::Mat m = cv::getAffineTransform(rect_pts, dst_pts);
return m;
}
// Exponential Moving Average (EMA) filter function
inline float EmaFilter(float currentProb, std::vector<float>& history, int max, float alpha = 0.2f) {
// Add current probability to history
history.push_back(currentProb);
// Trim history if it exceeds max size
if (history.size() > max) {
history.erase(history.begin(), history.begin() + (history.size() - max));
}
// Compute EMA
float ema = history[0]; // Initial value
for (size_t i = 1; i < history.size(); ++i) {
ema = alpha * history[i] + (1 - alpha) * ema;
}
return ema;
}
} // namespace inspire
#endif

5
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/age_predict.cpp
View File

@@ -1,5 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#include "age_predict.h"

14
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/age_predict.h
View File

@@ -1,14 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_AGEPREDICT_H
#define HYPERFACEREPO_AGEPREDICT_H
class AgePredict {
};
#endif //HYPERFACEREPO_AGEPREDICT_H

3
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/all.h
View File

@@ -6,7 +6,6 @@
#define HYPERFACEREPO_ATTRIBUTE_ALL_H
#include "mask_predict.h"
#include "gender_predict.h"
#include "age_predict.h"
#include "face_attribute.h"
#endif //HYPERFACEREPO_ATTRIBUTE_ALL_H

41
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute.cpp Normal file
View File

@@ -0,0 +1,41 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#include "face_attribute.h"
#include "middleware/utils.h"
namespace inspire {
FaceAttributePredict::FaceAttributePredict(): AnyNet("FaceAttributePredict") {}
std::vector<int> FaceAttributePredict::operator()(const Matrix& bgr_affine) {
AnyTensorOutputs outputs;
Forward(bgr_affine, outputs);
// cv::imshow("w", bgr_affine);
// cv::waitKey(0);
std::vector<float> &raceOut = outputs[0].second;
std::vector<float> &genderOut = outputs[1].second;
std::vector<float> &ageOut = outputs[2].second;
// for(int i = 0; i < raceOut.size(); i++) {
// std::cout << raceOut[i] << ", ";
// }
// std::cout << std::endl;
auto raceIdx = argmax(raceOut.begin(), raceOut.end());
auto genderIdx = argmax(genderOut.begin(), genderOut.end());
auto ageIdx = argmax(ageOut.begin(), ageOut.end());
std::string raceLabel = m_original_labels_[raceIdx];
std::string simplifiedLabel = m_label_map_.at(raceLabel);
int simplifiedRaceIdx = m_simplified_label_index_.at(simplifiedLabel);
// std::cout << raceLabel << std::endl;
// std::cout << simplifiedLabel << std::endl;
return {simplifiedRaceIdx, 1 - (int )genderIdx, (int )ageIdx};
}
} // namespace hyper

69
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute.h Normal file
View File

@@ -0,0 +1,69 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_GENDERPREDICT_H
#define HYPERFACEREPO_GENDERPREDICT_H
#include "data_type.h"
#include "middleware/any_net.h"
namespace inspire {
/**
* @class FaceAttributePredict
* @brief According to the face image, three classification information of age, gender and race were extracted.
*
* This class inherits from AnyNet and provides methods for performing face attribute prediction.
*/
class INSPIRE_API FaceAttributePredict : public AnyNet {
public:
/**
* @brief Constructor for FaceAttributePredict class.
*/
FaceAttributePredict();
/**
* @brief Exec infer.
*
* @param bgr_affine The BGR affine matrix to perform mask prediction on.
* @return The multi-list attribute prediction result.
*/
std::vector<int> operator()(const Matrix& bgr_affine);
private:
// Define primitive tag
const std::vector<std::string> m_original_labels_ = {
"Black", "East Asian", "Indian", "Latino_Hispanic", "Middle Eastern", "Southeast Asian", "White"
};
// Define simplified labels
const std::vector<std::string> m_simplified_labels_ = {
"Black", "Asian", "Latino/Hispanic", "Middle Eastern", "White"
};
// Define the mapping from the original tag to the simplified tag
const std::unordered_map<std::string, std::string> m_label_map_ = {
{"Black", "Black"},
{"East Asian", "Asian"},
{"Indian", "Asian"},
{"Latino_Hispanic", "Latino/Hispanic"},
{"Middle Eastern", "Middle Eastern"},
{"Southeast Asian", "Asian"},
{"White", "White"}
};
// Define index maps for simplified labels
const std::unordered_map<std::string, int> m_simplified_label_index_ = {
{"Black", 0},
{"Asian", 1},
{"Latino/Hispanic", 2},
{"Middle Eastern", 3},
{"White", 4}
};
};
} // namespace hyper
#endif //HYPERFACEREPO_GENDERPREDICT_H

6
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/gender_predict.cpp
View File

@@ -1,6 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#include "gender_predict.h"

14
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/gender_predict.h
View File

@@ -1,14 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_GENDERPREDICT_H
#define HYPERFACEREPO_GENDERPREDICT_H
class GenderPredict {
};
#endif //HYPERFACEREPO_GENDERPREDICT_H

125
cpp-package/inspireface/cpp/inspireface/pipeline_module/face_pipeline.cpp
View File

@@ -7,35 +7,31 @@
#include "log.h"
#include "track_module/landmark/face_landmark.h"
#include "recognition_module/extract/alignment.h"
#include "middleware/utils.h"
#include "herror.h"
namespace inspire {
FacePipeline::FacePipeline(InspireArchive &archive, bool enableLiveness, bool enableMaskDetect, bool enableAge,
bool enableGender, bool enableInteractionLiveness)
FacePipeline::FacePipeline(InspireArchive &archive, bool enableLiveness, bool enableMaskDetect, bool enableAttribute,
bool enableInteractionLiveness)
: m_enable_liveness_(enableLiveness),
m_enable_mask_detect_(enableMaskDetect),
m_enable_age_(enableAge),
m_enable_gender_(enableGender),
m_enable_attribute_(enableAttribute),
m_enable_interaction_liveness_(enableInteractionLiveness) {
if (m_enable_age_) {
InspireModel ageModel;
auto ret = InitAgePredict(ageModel);
if (m_enable_attribute_) {
InspireModel attrModel;
auto ret = archive.LoadModel("face_attribute", attrModel);
if (ret != 0) {
INSPIRE_LOGE("Load Face attribute model: %d", ret);
}
ret = InitFaceAttributePredict(attrModel);
if (ret != 0) {
INSPIRE_LOGE("InitAgePredict error.");
}
}
// Initialize the gender prediction model (assuming Index is 0)
if (m_enable_gender_) {
InspireModel genderModel;
auto ret = InitGenderPredict(genderModel);
if (ret != 0) {
INSPIRE_LOGE("InitGenderPredict error.");
}
}
// Initialize the mask detection model
if (m_enable_mask_detect_) {
InspireModel maskModel;
@@ -62,12 +58,17 @@ FacePipeline::FacePipeline(InspireArchive &archive, bool enableLiveness, bool en
}
}
// Initializing the model for in-vivo detection (assuming Index is 0)
// There may be a combination of algorithms for facial interaction
if (m_enable_interaction_liveness_) {
InspireModel actLivenessModel;
auto ret = InitLivenessInteraction(actLivenessModel);
// Blink model
InspireModel blinkModel;
auto ret = archive.LoadModel("blink_predict", blinkModel);
if (ret != 0) {
INSPIRE_LOGE("InitLivenessInteraction error.");
INSPIRE_LOGE("Load Blink model error.");
}
ret = InitBlinkFromLivenessInteraction(blinkModel);
if (ret != 0) {
INSPIRE_LOGE("InitBlinkFromLivenessInteraction error.");
}
}
@@ -75,6 +76,8 @@ FacePipeline::FacePipeline(InspireArchive &archive, bool enableLiveness, bool en
int32_t FacePipeline::Process(CameraStream &image, const HyperFaceData &face, FaceProcessFunction proc) {
cv::Mat originImage;
cv::Mat crop112;
switch (proc) {
case PROCESS_MASK: {
if (m_mask_predict_ == nullptr) {
@@ -91,12 +94,16 @@ int32_t FacePipeline::Process(CameraStream &image, const HyperFaceData &face, Fa
// }
// cv::imshow("wqwe", img);
// cv::waitKey(0);
auto trans = getTransformMatrix112(pointsFive);
trans.convertTo(trans, CV_64F);
auto crop = image.GetAffineRGBImage(trans, 112, 112);
if (crop112.empty())
{
auto trans = getTransformMatrix112(pointsFive);
trans.convertTo(trans, CV_64F);
crop112 = image.GetAffineRGBImage(trans, 112, 112);
}
// cv::imshow("wq", crop);
// cv::waitKey(0);
auto mask_score = (*m_mask_predict_)(crop);
auto mask_score = (*m_mask_predict_)(crop112);
faceMaskCache = mask_score;
break;
}
@@ -107,11 +114,12 @@ int32_t FacePipeline::Process(CameraStream &image, const HyperFaceData &face, Fa
// auto trans27 = getTransformMatrixSafas(pointsFive);
// trans27.convertTo(trans27, CV_64F);
// auto align112x27 = image.GetAffineRGBImage(trans27, 112, 112);
auto img = image.GetScaledImage(1.0, true);
if (originImage.empty()) {
originImage = image.GetScaledImage(1.0, true);
}
cv::Rect oriRect(face.rect.x, face.rect.y, face.rect.width, face.rect.height);
auto rect = GetNewBox(img.cols, img.rows, oriRect, 2.7f);
auto crop = img(rect);
auto rect = GetNewBox(originImage.cols, originImage.rows, oriRect, 2.7f);
auto crop = originImage(rect);
// cv::imwrite("crop.jpg", crop);
auto score = (*m_rgb_anti_spoofing_)(crop);
// auto i = cv::imread("zsb.jpg");
@@ -119,16 +127,45 @@ int32_t FacePipeline::Process(CameraStream &image, const HyperFaceData &face, Fa
faceLivenessCache = score;
break;
}
case PROCESS_AGE: {
if (m_age_predict_ == nullptr) {
case PROCESS_INTERACTION: {
if (m_blink_predict_ == nullptr) {
return HERR_SESS_PIPELINE_FAILURE; // uninitialized
}
if (originImage.empty()) {
originImage = image.GetScaledImage(1.0, true);
}
std::vector<std::vector<int>> order_list = {HLMK_LEFT_EYE_POINTS_INDEX, HLMK_RIGHT_EYE_POINTS_INDEX};
eyesStatusCache = {0, 0};
for (size_t i = 0; i < order_list.size(); i++)
{
const auto &index = order_list[i];
std::vector<cv::Point2f> points;
for (const auto &idx: index)
{
points.emplace_back(face.densityLandmark[idx].x, face.densityLandmark[idx].y);
}
cv::Rect2f rect = cv::boundingRect(points);
auto affine_scale = ComputeCropMatrix(rect, BlinkPredict::BLINK_EYE_INPUT_SIZE, BlinkPredict::BLINK_EYE_INPUT_SIZE);
affine_scale.convertTo(affine_scale, CV_64F);
auto pre_crop = image.GetAffineRGBImage(affine_scale, BlinkPredict::BLINK_EYE_INPUT_SIZE, BlinkPredict::BLINK_EYE_INPUT_SIZE);
auto eyeStatus = (*m_blink_predict_)(pre_crop);
eyesStatusCache[i] = eyeStatus;
}
break;
}
case PROCESS_GENDER: {
if (m_gender_predict_ == nullptr) {
case PROCESS_ATTRIBUTE: {
if (m_attribute_predict_ == nullptr) {
return HERR_SESS_PIPELINE_FAILURE; // uninitialized
}
std::vector<cv::Point2f> pointsFive;
for (const auto &p: face.keyPoints) {
pointsFive.push_back(HPointToPoint2f(p));
}
auto trans = getTransformMatrix112(pointsFive);
trans.convertTo(trans, CV_64F);
auto crop = image.GetAffineRGBImage(trans, 112, 112);
auto outputs = (*m_attribute_predict_)(crop);
faceAttributeCache = cv::Vec3i(outputs[0], outputs[1], outputs[2]);
break;
}
}
@@ -173,17 +210,16 @@ int32_t FacePipeline::Process(CameraStream &image, FaceObject &face) {
return HSUCCEED;
}
int32_t FacePipeline::InitAgePredict(InspireModel &) {
return 0;
int32_t FacePipeline::InitFaceAttributePredict(InspireModel &model) {
m_attribute_predict_ = std::make_shared<FaceAttributePredict>();
auto ret = m_attribute_predict_->loadData(model, model.modelType);
if (ret != InferenceHelper::kRetOk) {
return HERR_ARCHIVE_LOAD_FAILURE;
}
return HSUCCEED;
}
int32_t FacePipeline::InitGenderPredict(InspireModel &model) {
return 0;
}
int32_t FacePipeline::InitMaskPredict(InspireModel &model) {
m_mask_predict_ = std::make_shared<MaskPredict>();
auto ret = m_mask_predict_->loadData(model, model.modelType);
@@ -203,8 +239,13 @@ int32_t FacePipeline::InitRBGAntiSpoofing(InspireModel &model) {
return HSUCCEED;
}
int32_t FacePipeline::InitLivenessInteraction(InspireModel &model) {
return 0;
int32_t FacePipeline::InitBlinkFromLivenessInteraction(InspireModel &model) {
m_blink_predict_ = std::make_shared<BlinkPredict>();
auto ret = m_blink_predict_->loadData(model, model.modelType);
if (ret != InferenceHelper::kRetOk) {
return HERR_ARCHIVE_LOAD_FAILURE;
}
return HSUCCEED;
}
const std::shared_ptr<RBGAntiSpoofing> &FacePipeline::getMRgbAntiSpoofing() const {

41
cpp-package/inspireface/cpp/inspireface/pipeline_module/face_pipeline.h
View File

@@ -21,8 +21,8 @@ namespace inspire {
typedef enum FaceProcessFunction {
PROCESS_MASK = 0, ///< Mask detection.
PROCESS_RGB_LIVENESS, ///< RGB liveness detection.
PROCESS_AGE, ///< Age estimation.
PROCESS_GENDER, ///< Gender prediction.
PROCESS_ATTRIBUTE, ///< Face attribute estimation.
PROCESS_INTERACTION, ///< Face interaction.
} FaceProcessFunction;
/**
@@ -40,12 +40,11 @@ public:
* @param archive Model archive instance for model loading.
* @param enableLiveness Whether RGB liveness detection is enabled.
* @param enableMaskDetect Whether mask detection is enabled.
* @param enableAge Whether age estimation is enabled.
* @param enableGender Whether gender prediction is enabled.
* @param enableAttributee Whether face attribute estimation is enabled.
* @param enableInteractionLiveness Whether interaction liveness detection is enabled.
*/
explicit FacePipeline(InspireArchive &archive, bool enableLiveness, bool enableMaskDetect, bool enableAge,
bool enableGender, bool enableInteractionLiveness);
explicit FacePipeline(InspireArchive &archive, bool enableLiveness, bool enableMaskDetect, bool enableAttribute,
bool enableInteractionLiveness);
/**
* @brief Processes a face using the specified FaceProcessFunction.
@@ -70,20 +69,12 @@ public:
private:
/**
* @brief Initializes the AgePredict model.
* @brief Initializes the FaceAttributePredict model.
*
* @param model Pointer to the AgePredict model.
* @param model Pointer to the FaceAttributePredict model.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t InitAgePredict(InspireModel &model);
/**
* @brief Initializes the GenderPredict model.
*
* @param model Pointer to the GenderPredict model.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t InitGenderPredict(InspireModel &model);
int32_t InitFaceAttributePredict(InspireModel &model);
/**
* @brief Initializes the MaskPredict model.
@@ -102,29 +93,29 @@ private:
int32_t InitRBGAntiSpoofing(InspireModel &model);
/**
* @brief Initializes the LivenessInteraction model.
* @brief Initializes the Blink predict model.
*
* @param model Pointer to the LivenessInteraction model.
* @param model Pointer to the Blink predict model.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t InitLivenessInteraction(InspireModel &model);
int32_t InitBlinkFromLivenessInteraction(InspireModel &model);
private:
const bool m_enable_liveness_ = false; ///< Whether RGB liveness detection is enabled.
const bool m_enable_mask_detect_ = false; ///< Whether mask detection is enabled.
const bool m_enable_age_ = false; ///< Whether age estimation is enabled.
const bool m_enable_gender_ = false; ///< Whether gender prediction is enabled.
const bool m_enable_attribute_ = false; ///< Whether face attribute is enabled.
const bool m_enable_interaction_liveness_ = false; ///< Whether interaction liveness detection is enabled.
std::shared_ptr<AgePredict> m_age_predict_; ///< Pointer to AgePredict instance.
std::shared_ptr<GenderPredict> m_gender_predict_; ///< Pointer to GenderPredict instance.
std::shared_ptr<FaceAttributePredict> m_attribute_predict_; ///< Pointer to Face attribute prediction instance.
std::shared_ptr<MaskPredict> m_mask_predict_; ///< Pointer to MaskPredict instance.
std::shared_ptr<RBGAntiSpoofing> m_rgb_anti_spoofing_; ///< Pointer to RBGAntiSpoofing instance.
std::shared_ptr<LivenessInteraction> m_liveness_interaction_spoofing_; ///< Pointer to LivenessInteraction instance.
std::shared_ptr<BlinkPredict> m_blink_predict_; ///< Pointer to Blink predict instance.
public:
float faceMaskCache; ///< Cache for face mask detection result.
float faceLivenessCache; ///< Cache for face liveness detection result.
cv::Vec2f eyesStatusCache; ///< Cache for blink predict result.
cv::Vec3i faceAttributeCache; ///< Cache for face attribute predict result.
};
}

3
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/all.h
View File

@@ -5,7 +5,8 @@
#ifndef HYPERFACEREPO_LIVENESS_ALL_H
#define HYPERFACEREPO_LIVENESS_ALL_H
#include "liveness_interaction.h"
#include "blink_predict.h"
#include "rgb_anti_spoofing.h"
#include "order_of_hyper_landmark.h"
#endif //HYPERFACEREPO_LIVENESS_ALL_H

28
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/blink_predict.cpp Normal file
View File

@@ -0,0 +1,28 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#include "blink_predict.h"
#include "middleware/utils.h"
namespace inspire {
BlinkPredict::BlinkPredict(): AnyNet("BlinkPredict") {}
float BlinkPredict::operator()(const Matrix &bgr_affine) {
cv::Mat input;
if (bgr_affine.cols == BLINK_EYE_INPUT_SIZE && bgr_affine.rows == BLINK_EYE_INPUT_SIZE)
{
input = bgr_affine;
} else {
cv::resize(bgr_affine, input, cv::Size(BLINK_EYE_INPUT_SIZE, BLINK_EYE_INPUT_SIZE));
}
cv::cvtColor(input, input, cv::COLOR_BGR2GRAY);
AnyTensorOutputs outputs;
Forward(input, outputs);
auto &map = outputs[0].second;;
return map[1];
}
} // namespace inspire

41
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/blink_predict.h Normal file
View File

@@ -0,0 +1,41 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_BLINK_PREDICT_H
#define HYPERFACEREPO_BLINK_PREDICT_H
#include "data_type.h"
#include "middleware/any_net.h"
namespace inspire {
/**
* @class BlinkPredict
* @brief Prediction whether the eyes are open or closed.
*
* This class inherits from AnyNet and provides methods for performing blink prediction.
*/
class INSPIRE_API BlinkPredict : public AnyNet {
public:
/**
* @brief Constructor for MaskPredict class.
*/
BlinkPredict();
/**
* @brief Operator for performing blink prediction on a BGR affine matrix.
*
* @param bgr_affine The BGR affine matrix to perform mask prediction on.
* @return Blink prediction result.
*/
float operator()(const Matrix& bgr_affine);
public:
static const int BLINK_EYE_INPUT_SIZE = 64; ///< Input size
};
} // namespace inspire
#endif //HYPERFACEREPO_BLINK_PREDICT_H

5
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/liveness_interaction.cpp
View File

@@ -1,5 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#include "liveness_interaction.h"

14
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/liveness_interaction.h
View File

@@ -1,14 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_LIVENESSINTERACTION_H
#define HYPERFACEREPO_LIVENESSINTERACTION_H
class LivenessInteraction {
};
#endif //HYPERFACEREPO_LIVENESSINTERACTION_H

20
cpp-package/inspireface/cpp/inspireface/pipeline_module/liveness/order_of_hyper_landmark.h Normal file
View File

@@ -0,0 +1,20 @@
//
// Created by Tunm-Air13 on 2024/7/3.
//
#pragma once
#ifndef HYPERFACEREPO_ORDER_HYPERLANDMARK_H
#define HYPERFACEREPO_ORDER_HYPERLANDMARK_H
#include <iostream>
#include <vector>
namespace inspire {
// HyperLandmark left eye contour points sequence of dense facial landmarks.
const std::vector<int> HLMK_LEFT_EYE_POINTS_INDEX = {1, 34, 53, 59, 67, 3, 12, 94};
// HyperLandmark right eye contour points sequence of dense facial landmarks.
const std::vector<int> HLMK_RIGHT_EYE_POINTS_INDEX = {27, 104, 41, 85, 20, 47, 43, 51};
} // namespace inspire
#endif //HYPERFACEREPO_ORDER_HYPERLANDMARK_H

6
cpp-package/inspireface/cpp/inspireface/track_module/face_track.cpp
View File

@@ -174,7 +174,7 @@ bool FaceTrack::TrackFace(CameraStream &image, FaceObject &face) {
// pose and quality - BUG
auto rect = face.bbox_;
// std::cout << rect << std::endl;
auto affine_scale = FacePoseQuality::ComputeCropMatrix(rect);
auto affine_scale = ComputeCropMatrix(rect, FacePoseQuality::INPUT_WIDTH, FacePoseQuality::INPUT_HEIGHT);
affine_scale.convertTo(affine_scale, CV_64F);
auto pre_crop = image.GetAffineRGBImage(affine_scale, FacePoseQuality::INPUT_WIDTH,
FacePoseQuality::INPUT_HEIGHT);
@@ -245,7 +245,7 @@ void FaceTrack::UpdateStream(CameraStream &image) {
image.SetPreviewSize(track_preview_size_);
cv::Mat image_detect = image.GetPreviewImage(true);
nms();
for (auto const &face: trackingFace) {
cv::Rect m_mask_rect = face.GetRectSquare();
std::vector<cv::Point2f> pts = Rect2Points(m_mask_rect);
@@ -282,7 +282,7 @@ void FaceTrack::UpdateStream(CameraStream &image) {
}
}
nms();
// LOGD("Track Cost %f", t_track.GetCostTimeUpdate());
track_total_use_time_ = ((double) cv::getTickCount() - timeStart) / cv::getTickFrequency() * 1000;

20
cpp-package/inspireface/cpp/inspireface/track_module/quality/face_pose_quality.cpp
View File

@@ -35,26 +35,6 @@ FacePoseQualityResult FacePoseQuality::operator()(const Matrix &bgr_affine) {
return res;
}
cv::Mat FacePoseQuality::ComputeCropMatrix(const cv::Rect2f &rect) {
float x = rect.x;
float y = rect.y;
float w = rect.width;
float h = rect.height;
float cx = x + w / 2;
float cy = y + h / 2;
float length = std::max(w, h) * 1.5 / 2;
float x1 = cx - length;
float y1 = cy - length;
float x2 = cx + length;
float y2 = cy + length;
cv::Rect2f padding_rect(x1, y1, x2 - x1, y2 - y1);
std::vector<cv::Point2f> rect_pts = Rect2Points(padding_rect);
rect_pts.erase(rect_pts.end() - 1);
std::vector<cv::Point2f> dst_pts = {{0, 0}, {INPUT_WIDTH, 0}, {INPUT_WIDTH, INPUT_HEIGHT}};
cv::Mat m = cv::getAffineTransform(rect_pts, dst_pts);
return m;
}
} // namespace hyper

6
cpp-package/inspireface/cpp/inspireface/track_module/quality/face_pose_quality.h
View File

@@ -44,12 +44,6 @@ public:
*/
FacePoseQualityResult operator()(const Matrix& bgr_affine);
/**
* @brief Computes the affine transformation matrix for face cropping.
* @param rect Rectangle representing the face in the image.
* @return cv::Mat The computed affine transformation matrix.
*/
static cv::Mat ComputeCropMatrix(const cv::Rect2f &rect);
public:
const static int INPUT_WIDTH = 96; ///< Width of the input image for the network.

2
cpp-package/inspireface/cpp/inspireface/version.txt
View File

@@ -1 +1 @@
InspireFace Version: 1.1.2
InspireFace Version: 1.1.4

61
cpp-package/inspireface/cpp/sample/cpp/sample_face_track.cpp
View File

@@ -7,16 +7,27 @@
int main(int argc, char* argv[]) {
// Check whether the number of parameters is correct
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <pack_path> <source_path>\n";
if (argc < 3 || argc > 4) {
std::cerr << "Usage: " << argv[0] << " <pack_path> <source_path> [rotation]\n";
return 1;
}
auto packPath = argv[1];
auto sourcePath = argv[2];
int rotation = 0;
// If rotation is provided, check and set the value
if (argc == 4) {
rotation = std::atoi(argv[3]);
if (rotation != 0 && rotation != 90 && rotation != 180 && rotation != 270) {
std::cerr << "Invalid rotation value. Allowed values are 0, 90, 180, 270.\n";
return 1;
}
}
std::cout << "Pack file Path: " << packPath << std::endl;
std::cout << "Source file Path: " << sourcePath << std::endl;
std::cout << "Rotation: " << rotation << std::endl;
HResult ret;
// The resource file must be loaded before it can be used
@@ -55,9 +66,26 @@ int main(int argc, char* argv[]) {
HFImageData imageParam = {0};
imageParam.data = image.data; // Data buffer
imageParam.width = image.cols; // Target view width
imageParam.height = image.rows; // Target view width
imageParam.rotation = HF_CAMERA_ROTATION_0; // Data source rotate
imageParam.format = HF_STREAM_BGR; // Data source format
imageParam.height = image.rows; // Target view width
// Set rotation based on input parameter
switch (rotation) {
case 90:
imageParam.rotation = HF_CAMERA_ROTATION_90;
break;
case 180:
imageParam.rotation = HF_CAMERA_ROTATION_180;
break;
case 270:
imageParam.rotation = HF_CAMERA_ROTATION_270;
break;
case 0:
default:
imageParam.rotation = HF_CAMERA_ROTATION_0;
break;
}
imageParam.format = HF_STREAM_BGR; // Data source format
// Create an image data stream
HFImageStream imageHandle = {0};
@@ -82,10 +110,11 @@ int main(int argc, char* argv[]) {
cv::Mat draw = image.clone();
for (int index = 0; index < faceNum; ++index) {
std::cout << "========================================" << std::endl;
std::cout << "Token size: " << multipleFaceData.tokens[index].size << std::endl;
std::cout << "Process face index: " << index << std::endl;
// Use OpenCV's Rect to receive face bounding boxes
auto rect = cv::Rect(multipleFaceData.rects[index].x, multipleFaceData.rects[index].y,
multipleFaceData.rects[index].width, multipleFaceData.rects[index].height);
multipleFaceData.rects[index].width, multipleFaceData.rects[index].height);
cv::rectangle(draw, rect, cv::Scalar(0, 100, 255), 4);
// Print FaceID, In IMAGE-MODE it is changing, in VIDEO-MODE it is fixed, but it may be lost
@@ -93,9 +122,21 @@ int main(int argc, char* argv[]) {
// Print Head euler angle, It can often be used to judge the quality of a face by the Angle of the head
std::cout << "Roll: " << multipleFaceData.angles.roll[index]
<< ", Yaw: " << multipleFaceData.angles.roll[index]
<< ", Pitch: " << multipleFaceData.angles.pitch[index] << std::endl;
<< ", Yaw: " << multipleFaceData.angles.roll[index]
<< ", Pitch: " << multipleFaceData.angles.pitch[index] << std::endl;
HInt32 numOfLmk;
HFGetNumOfFaceDenseLandmark(&numOfLmk);
HPoint2f denseLandmarkPoints[numOfLmk];
ret = HFGetFaceDenseLandmarkFromFaceToken(multipleFaceData.tokens[index], denseLandmarkPoints, numOfLmk);
if (ret != HSUCCEED) {
std::cerr << "HFGetFaceDenseLandmarkFromFaceToken error!!" << std::endl;
return -1;
}
for (size_t i = 0; i < numOfLmk; i++) {
cv::Point2f p(denseLandmarkPoints[i].x, denseLandmarkPoints[i].y);
cv::circle(draw, p, 0, (0, 0, 255), 2);
}
}
cv::imwrite("draw_detected.jpg", draw);
@@ -117,7 +158,6 @@ int main(int argc, char* argv[]) {
return -1;
}
// Get face quality results from the pipeline cache
HFFaceQualityConfidence qualityConfidence = {0};
ret = HFGetFaceQualityConfidence(session, &qualityConfidence);
@@ -152,6 +192,5 @@ int main(int argc, char* argv[]) {
return ret;
}
return 0;
}
}

39
cpp-package/inspireface/cpp/sample/cpp/sample_face_track_video.cpp
View File

@@ -44,13 +44,13 @@ int main(int argc, char* argv[]) {
}
// Enable the functions in the pipeline: mask detection, live detection, and face quality detection
HOption option = HF_ENABLE_QUALITY | HF_ENABLE_MASK_DETECT | HF_ENABLE_LIVENESS;
HOption option = HF_ENABLE_QUALITY | HF_ENABLE_MASK_DETECT | HF_ENABLE_INTERACTION;
// Video or frame sequence mode uses VIDEO-MODE, which is face detection with tracking
HFDetectMode detMode = HF_DETECT_MODE_TRACK_BY_DETECTION;
HFDetectMode detMode = HF_DETECT_MODE_LIGHT_TRACK;
// Maximum number of faces detected
HInt32 maxDetectNum = 20;
// Face detection image input level
HInt32 detectPixelLevel = 320;
HInt32 detectPixelLevel = 160;
// fps in tracking-by-detection mode
HInt32 trackByDetectFps = 20;
HFSession session = {0};
@@ -122,7 +122,25 @@ int main(int argc, char* argv[]) {
// Draw detection mode on the frame
drawMode(draw, detMode);
if (faceNum > 0) {
ret = HFMultipleFacePipelineProcessOptional(session, imageHandle, &multipleFaceData, option);
if (ret != HSUCCEED)
{
std::cout << "HFMultipleFacePipelineProcessOptional error: " << ret << std::endl;
return ret;
}
HFFaceIntereactionResult result;
ret = HFGetFaceIntereactionResult(session, &result);
if (ret != HSUCCEED)
{
std::cout << "HFGetFaceIntereactionResult error: " << ret << std::endl;
return ret;
}
std::cout << "Left eye status: " << result.leftEyeStatusConfidence[0] << std::endl;
std::cout << "Righ eye status: " << result.rightEyeStatusConfidence[0] << std::endl;
}
for (int index = 0; index < faceNum; ++index) {
// std::cout << "========================================" << std::endl;
// std::cout << "Process face index: " << index << std::endl;
@@ -143,8 +161,21 @@ int main(int argc, char* argv[]) {
// Add TrackID to the drawing
cv::putText(draw, "ID: " + std::to_string(trackId), cv::Point(rect.x, rect.y - 10),
cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 2);
}
HInt32 numOfLmk;
HFGetNumOfFaceDenseLandmark(&numOfLmk);
HPoint2f denseLandmarkPoints[numOfLmk];
ret = HFGetFaceDenseLandmarkFromFaceToken(multipleFaceData.tokens[index], denseLandmarkPoints, numOfLmk);
if (ret != HSUCCEED) {
std::cerr << "HFGetFaceDenseLandmarkFromFaceToken error!!" << std::endl;
return -1;
}
for (size_t i = 0; i < numOfLmk; i++) {
cv::Point2f p(denseLandmarkPoints[i].x, denseLandmarkPoints[i].y);
cv::circle(draw, p, 0, (0, 0, 255), 2);
}
}
cv::imshow("w", draw);
cv::waitKey(1);

2
cpp-package/inspireface/cpp/test/test.cpp
View File

@@ -100,7 +100,7 @@ int main(int argc, char* argv[]) {
}
// Set log level
HFSetLogLevel(HF_LOG_ERROR);
HFSetLogLevel(HF_LOG_INFO);
return session.run();
}

223
cpp-package/inspireface/cpp/test/unit/api/test_face_pipeline.cpp
View File

@@ -7,6 +7,115 @@
#include "inspireface/c_api/inspireface.h"
#include "../test_helper/test_tools.h"
TEST_CASE("test_FacePipelineAttribute", "[face_pipeline_attribute]") {
DRAW_SPLIT_LINE
TEST_PRINT_OUTPUT(true);
enum AGE_BRACKED {
AGE_0_2 = 0, ///< Age 0-2 years old
AGE_3_9, ///< Age 3-9 years old
AGE_10_19, ///< Age 10-19 years old
AGE_20_29, ///< Age 20-29 years old
AGE_30_39, ///< Age 30-39 years old
AGE_40_49, ///< Age 40-49 years old
AGE_50_59, ///< Age 50-59 years old
AGE_60_69, ///< Age 60-69 years old
MORE_THAN_70, ///< Age more than 70 years old
};
enum GENDER {
FEMALE = 0, ///< Female
MALE, ///< Male
};
enum RACE {
BLACK = 0, ///< Black
ASIAN, ///< Asian
LATINO_HISPANIC, ///< Latino/Hispanic
MIDDLE_EASTERN, ///< Middle Eastern
WHITE, ///< White
};
HResult ret;
HFSessionCustomParameter parameter = {0};
parameter.enable_face_attribute = 1;
HFDetectMode detMode = HF_DETECT_MODE_ALWAYS_DETECT;
HFSession session;
HInt32 faceDetectPixelLevel = 160;
ret = HFCreateInspireFaceSession(parameter, detMode, 5, faceDetectPixelLevel, -1, &session);
REQUIRE(ret == HSUCCEED);
SECTION("a black girl") {
HFImageStream imgHandle;
auto img = cv::imread(GET_DATA("data/attribute/1423.jpg"));
REQUIRE(!img.empty());
ret = CVImageToImageStream(img, imgHandle);
REQUIRE(ret == HSUCCEED);
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum == 1);
// Run pipeline
ret = HFMultipleFacePipelineProcessOptional(session, imgHandle, &multipleFaceData, HF_ENABLE_FACE_ATTRIBUTE);
REQUIRE(ret == HSUCCEED);
HFFaceAttributeResult result = {0};
ret = HFGetFaceAttributeResult(session, &result);
REQUIRE(ret == HSUCCEED);
REQUIRE(result.num == 1);
// Check attribute
CHECK(result.race[0] == BLACK);
CHECK(result.ageBracket[0] == AGE_10_19);
CHECK(result.gender[0] == FEMALE);
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
imgHandle = nullptr;
}
SECTION("two young white women") {
HFImageStream imgHandle;
auto img = cv::imread(GET_DATA("data/attribute/7242.jpg"));
REQUIRE(!img.empty());
ret = CVImageToImageStream(img, imgHandle);
REQUIRE(ret == HSUCCEED);
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum == 2);
// Run pipeline
ret = HFMultipleFacePipelineProcessOptional(session, imgHandle, &multipleFaceData, HF_ENABLE_FACE_ATTRIBUTE);
REQUIRE(ret == HSUCCEED);
HFFaceAttributeResult result = {0};
ret = HFGetFaceAttributeResult(session, &result);
REQUIRE(ret == HSUCCEED);
REQUIRE(result.num == 2);
// Check attribute
for (size_t i = 0; i < result.num; i++)
{
CHECK(result.race[i] == WHITE);
CHECK(result.ageBracket[i] == AGE_20_29);
CHECK(result.gender[i] == FEMALE);
}
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
imgHandle = nullptr;
}
ret = HFReleaseInspireFaceSession(session);
session = nullptr;
REQUIRE(ret == HSUCCEED);
}
TEST_CASE("test_FacePipeline", "[face_pipeline]") {
DRAW_SPLIT_LINE
TEST_PRINT_OUTPUT(true);
@@ -184,8 +293,120 @@ TEST_CASE("test_FacePipeline", "[face_pipeline]") {
ret = HFReleaseInspireFaceSession(session);
REQUIRE(ret == HSUCCEED);
}
}
TEST_CASE("test_FaceReaction", "[face_reaction]") {
DRAW_SPLIT_LINE
TEST_PRINT_OUTPUT(true);
HResult ret;
HFSessionCustomParameter parameter = {0};
parameter.enable_interaction_liveness = 1;
parameter.enable_liveness = 1;
HFDetectMode detMode = HF_DETECT_MODE_ALWAYS_DETECT;
HFSession session;
ret = HFCreateInspireFaceSession(parameter, detMode, 3, -1, -1, &session);
REQUIRE(ret == HSUCCEED);
SECTION("open eyes") {
// Get a face picture
HFImageStream imgHandle;
auto img = cv::imread(GET_DATA("data/reaction/open_eyes.png"));
ret = CVImageToImageStream(img, imgHandle);
REQUIRE(ret == HSUCCEED);
// Extract basic face information from photos
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum > 0);
// Predict eyes status
ret = HFMultipleFacePipelineProcess(session, imgHandle, &multipleFaceData, parameter);
REQUIRE(ret == HSUCCEED);
// Get results
HFFaceIntereactionResult result;
ret = HFGetFaceIntereactionResult(session, &result);
REQUIRE(multipleFaceData.detectedNum == result.num);
REQUIRE(ret == HSUCCEED);
// Check
CHECK(result.leftEyeStatusConfidence[0] > 0.5f);
CHECK(result.rightEyeStatusConfidence[0] > 0.5f);
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
}
}
SECTION("close eyes") {
// Get a face picture
HFImageStream imgHandle;
auto img = cv::imread(GET_DATA("data/reaction/close_eyes.jpeg"));
ret = CVImageToImageStream(img, imgHandle);
REQUIRE(ret == HSUCCEED);
// Extract basic face information from photos
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum > 0);
// Predict eyes status
ret = HFMultipleFacePipelineProcess(session, imgHandle, &multipleFaceData, parameter);
REQUIRE(ret == HSUCCEED);
// Get results
HFFaceIntereactionResult result;
ret = HFGetFaceIntereactionResult(session, &result);
REQUIRE(multipleFaceData.detectedNum == result.num);
REQUIRE(ret == HSUCCEED);
// Check
CHECK(result.leftEyeStatusConfidence[0] < 0.5f);
CHECK(result.rightEyeStatusConfidence[0] < 0.5f);
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
}
SECTION("Close one eye and open the other") {
// Get a face picture
HFImageStream imgHandle;
auto img = cv::imread(GET_DATA("data/reaction/close_open_eyes.jpeg"));
ret = CVImageToImageStream(img, imgHandle);
REQUIRE(ret == HSUCCEED);
// Extract basic face information from photos
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum > 0);
// Predict eyes status
ret = HFMultipleFacePipelineProcess(session, imgHandle, &multipleFaceData, parameter);
REQUIRE(ret == HSUCCEED);
// Get results
HFFaceIntereactionResult result;
ret = HFGetFaceIntereactionResult(session, &result);
REQUIRE(multipleFaceData.detectedNum == result.num);
REQUIRE(ret == HSUCCEED);
// Check
CHECK(result.leftEyeStatusConfidence[0] < 0.5f);
CHECK(result.rightEyeStatusConfidence[0] > 0.5f);
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
}
ret = HFReleaseInspireFaceSession(session);
REQUIRE(ret == HSUCCEED);
}

56
cpp-package/inspireface/cpp/test/unit/api/test_face_track.cpp
View File

@@ -483,4 +483,60 @@ TEST_CASE("test_MultipleLevelFaceDetect", "[face_detect]") {
}
}
TEST_CASE("test_FaceShowLandmark", "[face_landmark]") {
DRAW_SPLIT_LINE
TEST_PRINT_OUTPUT(true);
std::vector<std::string> images_path = {
GET_DATA("data/reaction/close_open_eyes.jpeg"),
GET_DATA("data/reaction/open_eyes.png"),
GET_DATA("data/reaction/close_eyes.jpeg"),
};
HResult ret;
HFSessionCustomParameter parameter = {0};
HFDetectMode detMode = HF_DETECT_MODE_ALWAYS_DETECT;
HFSession session;
HInt32 detectPixelLevel = 160;
ret = HFCreateInspireFaceSession(parameter, detMode, 20, detectPixelLevel, -1, &session);
REQUIRE(ret == HSUCCEED);
HFSessionSetTrackPreviewSize(session, detectPixelLevel);
HFSessionSetFilterMinimumFacePixelSize(session, 0);
for (size_t i = 0; i < images_path.size(); i++)
{
HFImageStream imgHandle;
auto image = cv::imread(images_path[i]);
ret = CVImageToImageStream(image, imgHandle);
REQUIRE(ret == HSUCCEED);
// Extract basic face information from photos
HFMultipleFaceData multipleFaceData = {0};
ret = HFExecuteFaceTrack(session, imgHandle, &multipleFaceData);
REQUIRE(ret == HSUCCEED);
REQUIRE(multipleFaceData.detectedNum > 0);
HInt32 numOfLmk;
HFGetNumOfFaceDenseLandmark(&numOfLmk);
HPoint2f denseLandmarkPoints[numOfLmk];
ret = HFGetFaceDenseLandmarkFromFaceToken(multipleFaceData.tokens[0], denseLandmarkPoints, numOfLmk);
REQUIRE(ret == HSUCCEED);
for (size_t i = 0; i < numOfLmk; i++) {
cv::Point2f p(denseLandmarkPoints[i].x, denseLandmarkPoints[i].y);
cv::circle(image, p, 0, (0, 0, 255), 2);
}
cv::imwrite("lml_" + std::to_string(i) + ".jpg", image);
ret = HFReleaseImageStream(imgHandle);
REQUIRE(ret == HSUCCEED);
}
ret = HFReleaseInspireFaceSession(session);
REQUIRE(ret == HSUCCEED);
}