ZF/insightface
1
0
Fork 0
mirror of https://github.com/deepinsight/insightface.git synced 2026-08-23 21:57:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update inspireface to 1.2.0

Browse Source
This commit is contained in:
Jingyu
2025-03-25 00:51:26 +08:00
parent 977ea6795b
commit ca64996b84
388 changed files with 28584 additions and 13036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
cpp-package/inspireface/cpp/inspireface/CMakeLists.txt
View File

@@ -6,41 +6,79 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
option(ISF_BUILD_SHARED_LIBS "Build shared libraries (DLLs)." ON)
option(ISF_ENABLE_TRACKING_BY_DETECTION "Use the tracking-by-detection mode." OFF)
if(ISF_ENABLE_TRACKING_BY_DETECTION)
add_definitions("-DISF_ENABLE_TRACKING_BY_DETECTION")
find_package(Eigen3 REQUIRED)
include_directories(${EIGEN3_INCLUDE_DIRS})
string(TIMESTAMP BUILD_TIMESTAMP "%Y-%m-%d")
set(EXTENDED_INFORMATION "InspireFace[Community Edition]")
if(INSPIRECV_BACKEND_OPENCV)
set(EXTENDED_INFORMATION "${EXTENDED_INFORMATION}@OpenCV Backend")
else()
set(EXTENDED_INFORMATION "${EXTENDED_INFORMATION}@General")
endif()
set(EXTENDED_INFORMATION "${EXTENDED_INFORMATION} - Build Time: ${BUILD_TIMESTAMP}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/information.h.in ${CMAKE_CURRENT_SOURCE_DIR}/information.h)
file(GLOB_RECURSE SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/c_api/inspireface.cc) # Add C_API file
if (ISF_ENABLE_RKNN)
set(ISF_RKNN_API_INCLUDE_DIRS ${ISF_THIRD_PARTY_DIR}/inspireface-precompile/rknn/${ISF_RKNPU_MAJOR}/runtime/${ISF_RK_DEVICE_TYPE}/Linux/librknn_api/include)
set(ISF_RKNN_API_LIB ${ISF_THIRD_PARTY_DIR}/inspireface-precompile/rknn/${ISF_RKNPU_MAJOR}/runtime/${ISF_RK_DEVICE_TYPE}/Linux/librknn_api/${CPU_ARCH}/)
link_directories(${ISF_RKNN_API_LIB})
if (ISF_RKNPU_MAJOR STREQUAL "rknpu1")
set(ISF_RKNN_API_INCLUDE_DIRS ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${ISF_RK_DEVICE_TYPE}/Linux/librknn_api/include)
set(ISF_RKNN_API_LIB ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${ISF_RK_DEVICE_TYPE}/Linux/librknn_api/${CPU_ARCH}/)
link_directories(${ISF_RKNN_API_LIB})
set(RKNN_LINKED rknn_api)
else()
if(ANDROID)
set(RK_PLATFORM "Android")
set(ISF_RKNN_API_INCLUDE_DIRS ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${RK_PLATFORM}/librknn_api/include)
set(ISF_RKNN_API_LIB ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${RK_PLATFORM}/librknn_api/${ANDROID_ABI}/)
link_directories(${ISF_RKNN_API_LIB})
set(RKNN_LINKED rknnrt)
else()
set(RK_PLATFORM "Linux")
set(ISF_RKNN_API_INCLUDE_DIRS ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${RK_PLATFORM}/librknn_api/include)
if (ISF_RK_COMPILER_TYPE STREQUAL "aarch64")
set(ISF_RKNN_API_LIB ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${RK_PLATFORM}/librknn_api/${ISF_RK_COMPILER_TYPE}/)
link_directories(${ISF_RKNN_API_LIB})
else()
# For rknpu2 with armv7, we recommend linking static libraries by default
set(ISF_RKNN_API_LIB ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/rknn/${ISF_RKNPU_MAJOR}/runtime/${RK_PLATFORM}/librknn_api/${ISF_RK_COMPILER_TYPE}/librknnmrt.a)
set(RKNN_LINKED ${ISF_RKNN_API_LIB})
set(RKNN_USE_STATIC_LIBS TRUE)
endif()
endif()
endif()
endif()
set(LINK_THIRD_LIBS ${MNN_LIBS})
# OpenCV
set(LINK_THIRD_LIBS ${OpenCV_LIBS} ${MNN_LIBS})
if(ISF_ENABLE_TRACKING_BY_DETECTION)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} Eigen3::Eigen)
if(ISF_ENABLE_OPENCV)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${OpenCV_LIBS})
endif()
# SQLite3
set(SOURCE_FILES ${SOURCE_FILES} ${ISF_THIRD_PARTY_DIR}/inspireface-precompile/sqlite/sqlite3.c) # Add SQLite3 C_API file
set(SQLITE_INCLUDE ${ISF_THIRD_PARTY_DIR}/inspireface-precompile/sqlite/)
set(SOURCE_FILES ${SOURCE_FILES}
${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/sqlite/sqlite3.c
${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/sqlite/sqlite-vec.c
) # Add SQLite3 C_API file
set(SQLITE_INCLUDE ${ISF_THIRD_PARTY_DIR}/inspireface-precompile-lite/sqlite/)
if (ISF_ENABLE_RKNN)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} rknn_api dl)
# InferenceHelp use RkNN
add_definitions("-DINFERENCE_HELPER_ENABLE_RKNN")
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${RKNN_LINKED} dl)
# InferenceWrapper use RkNN
if (ISF_RKNPU_MAJOR STREQUAL "rknpu1")
add_definitions("-DINFERENCE_WRAPPER_ENABLE_RKNN")
elseif(ISF_RKNPU_MAJOR STREQUAL "rknpu2")
add_definitions("-DINFERENCE_WRAPPER_ENABLE_RKNN2")
endif()
if (ISF_RK_COMPILER_TYPE STREQUAL "aarch64")
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} rknnrt)
endif()
endif()
# cpp yaml
@@ -49,25 +87,45 @@ set(SOURCE_FILES ${SOURCE_FILES} ${CPP_YAML_SRC})
set(CPP_YAML_INCLUDE ${ISF_THIRD_PARTY_DIR}/yaml-cpp/include)
# archive
set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_archive/microtar/microtar.c)
set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_archive/core_archive/microtar/microtar.c)
# CoreArchive source file
set(CORE_ARCHIVE_SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_archive/core_archive/core_archive.cc CACHE PATH "")
set(SOURCE_FILES ${SOURCE_FILES} ${CORE_ARCHIVE_SOURCE_FILE})
if(ISF_ENABLE_APPLE_EXTENSION)
# link apple libs
find_library(FOUNDATION_LIBRARY Foundation)
find_library(COREML_LIBRARY CoreML)
find_library(ACCELERATE_LIBRARY Accelerate)
find_package(OpenCV REQUIRED)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${FOUNDATION_LIBRARY} ${COREML_LIBRARY} ${ACCELERATE_LIBRARY})
# Add objective-c files
set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/inference_wrapper/coreml/CoreMLAdapter.mm)
endif()
# MNN
link_directories(${MNN_LIBS})
if(ISF_BUILD_SHARED_LIBS)
add_definitions("-DISF_BUILD_SHARED_LIBS")
add_library(InspireFace SHARED ${SOURCE_FILES})
add_library(InspireFace SHARED ${SOURCE_FILES} $<TARGET_OBJECTS:inspirecv>)
else()
add_library(InspireFace STATIC ${SOURCE_FILES})
add_library(InspireFace STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:inspirecv>)
endif()
target_compile_definitions(InspireFace PUBLIC INFERENCE_HELPER_ENABLE_MNN)
target_compile_definitions(InspireFace PUBLIC INFERENCE_WRAPPER_ENABLE_MNN)
target_compile_definitions(InspireFace PUBLIC FEATURE_BLOCK_ENABLE_OPENCV)
# Include files
set(NEED_INCLUDE . ${MNN_INCLUDE_DIRS})
if (ISF_ENABLE_RKNN)
set(NEED_INCLUDE ${NEED_INCLUDE} ${ISF_RKNN_API_INCLUDE_DIRS})
if(ISF_ENABLE_RGA)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${RGA_LIBS})
set(NEED_INCLUDE ${NEED_INCLUDE} ${RGA_INCLUDE_DIRS})
endif()
endif ()
if (ISF_BUILD_LINUX_ARM7 OR ANDROID)
@@ -78,6 +136,12 @@ if (ISF_BUILD_LINUX_ARM7 OR ISF_BUILD_LINUX_AARCH64)
set(NEED_INCLUDE ${NEED_INCLUDE} ${OpenCV_STATIC_INCLUDE_DIR})
endif ()
if (ISF_ENABLE_TENSORRT)
set(NEED_INCLUDE ${NEED_INCLUDE} ${ISF_TENSORRT_INCLUDE_DIRS})
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${ISF_TENSORRT_LIBRARIES})
endif()
# add cpp yaml header
set(NEED_INCLUDE ${NEED_INCLUDE} ${CPP_YAML_INCLUDE} ${SQLITE_INCLUDE})
@@ -86,6 +150,9 @@ if(PLAT STREQUAL "linux")
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${CMAKE_THREAD_LIBS_INIT} dl)
endif()
# set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} inspirecv)
set(NEED_INCLUDE ${NEED_INCLUDE} ${INSPIRECV_INCLUDE_PATH})
target_include_directories(InspireFace PUBLIC
${NEED_INCLUDE}
)
@@ -101,49 +168,65 @@ elseif(IOS)
# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ObjC")
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ObjC")
target_link_libraries(InspireFace
"-fobjc-arc"
"-framework opencv2"
"-framework Metal"
"-framework CoreML"
"-framework Foundation"
"-framework CoreVideo"
"-framework CoreMedia"
${MNN_FRAMEWORK_PATH}
)
"-fobjc-arc"
"-framework Metal"
"-framework CoreML"
"-framework Foundation"
"-framework CoreVideo"
"-framework CoreMedia"
${MNN_FRAMEWORK_PATH}
)
if(ISF_ENABLE_APPLE_EXTENSION)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} ${ACCELERATE_LIBRARY})
endif()
if(ISF_ENABLE_OPENCV)
set(LINK_THIRD_LIBS ${LINK_THIRD_LIBS} "-framework opencv2")
endif()
else()
target_link_libraries(InspireFace PUBLIC ${LINK_THIRD_LIBS})
set_target_properties(InspireFace PROPERTIES
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/
)
)
endif()
# Print Message
message(STATUS ">>>>>>>>>>>>>")
message(STATUS "InspireFace Core:")
message(STATUS "\t Version: ${INSPIRE_FACE_VERSION_MAJOR}.${INSPIRE_FACE_VERSION_MINOR}.${INSPIRE_FACE_VERSION_PATCH}")
message(STATUS "\t MNN_INCLUDE_DIRS: ${MNN_INCLUDE_DIRS}")
message(STATUS "\t MNN_LIBS: ${MNN_LIBS}")
message(STATUS "\t ENABLE_TRACKING_BY_DETECTION: ${ISF_ENABLE_TRACKING_BY_DETECTION}")
if(ISF_ENABLE_TRACKING_BY_DETECTION)
message(STATUS "\t EIGEN3_PATH: ${EIGEN3_INCLUDE_DIRS}")
endif()
message(STATUS "\t ISF_BUILD_SHARED_LIBS: ${ISF_BUILD_SHARED_LIBS}")
message(STATUS "\t ISF_ENABLE_RKNN: ${ISF_ENABLE_RKNN}")
if (ISF_ENABLE_RKNN)
message(STATUS "\t ISF_RKNN_API_INCLUDE_DIRS: ${ISF_RKNN_API_INCLUDE_DIRS}")
message(STATUS "\t ISF_RKNN_API_LIB: ${ISF_RKNN_API_LIB}")
endif ()
if (ISF_ENABLE_RGA)
message(STATUS "\t ISF_ENABLE_RGA: ${ISF_ENABLE_RGA}")
message(STATUS "\t RGA_LIBS: ${RGA_LIBS}")
message(STATUS "\t RGA_INCLUDE_DIRS: ${RGA_INCLUDE_DIRS}")
endif ()
if (ISF_GLOBAL_INFERENCE_BACKEND_USE_MNN_CUDA)
message(STATUS "\t ISF_GLOBAL_INFERENCE_BACKEND_USE_MNN_CUDA: ${ISF_GLOBAL_INFERENCE_BACKEND_USE_MNN_CUDA}")
endif ()
message(STATUS "\t ISF_ENABLE_TENSORRT: ${ISF_ENABLE_TENSORRT}")
if (ISF_ENABLE_TENSORRT)
message(STATUS "\t TENSORRT_INCLUDE_DIR: ${TENSORRT_INCLUDE_DIR}")
message(STATUS "\t TENSORRT_LIBRARY_INFER: ${TENSORRT_LIBRARY_INFER}")
message(STATUS "\t TENSORRT_LIBRARY_RUNTIME: ${TENSORRT_LIBRARY_RUNTIME}")
message(STATUS "\t CUDA_RUNTIME_LIBRARY: ${CUDA_RUNTIME_LIBRARY}")
endif()
# Install lib
install(TARGETS InspireFace
LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib
ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib
)
)
# Install header file
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/c_api/inspireface.h DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/include)
@@ -154,8 +237,32 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/herror.h DESTINATION ${CMAKE_INSTALL_P
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/version.txt DESTINATION ${CMAKE_INSTALL_PREFIX}/)
if (ISF_ENABLE_RKNN)
if (ISF_ENABLE_RKNN AND ISF_RKNPU_MAJOR STREQUAL "rknpu1")
# Install rknn 3rd lib
install(FILES ${ISF_RKNN_API_LIB}/librknn_api.so DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
endif ()
if(ISF_ENABLE_RKNN AND ISF_RKNPU_MAJOR STREQUAL "rknpu2" AND RK_PLATFORM STREQUAL "Android")
install(FILES ${ISF_RKNN_API_LIB}/librknnrt.so DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
endif()
if (ISF_RK_COMPILER_TYPE STREQUAL "aarch64")
install(FILES ${ISF_RKNN_API_LIB}/librknnrt.so DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
endif()
if (NOT IOS)
if (NOT ISF_BUILD_SHARED_LIBS)
if(MNN_BUILD_SHARED_LIBS)
install(FILES ${CMAKE_BINARY_DIR}/3rdparty/MNN/libMNN.so DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
else()
install(FILES ${CMAKE_BINARY_DIR}/3rdparty/MNN/libMNN.a DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
endif()
if(RKNN_USE_STATIC_LIBS)
# To be added: The compilation of the RK series needs to be added
install(FILES ${ISF_RKNN_API_LIB} DESTINATION ${CMAKE_INSTALL_PREFIX}/InspireFace/lib)
endif()
endif()
endif()

147
cpp-package/inspireface/cpp/inspireface/Initialization_module/launch.cpp
View File

@@ -1,10 +1,18 @@
//
// Created by tunm on 2024/4/17.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "launch.h"
#include "log.h"
#include "herror.h"
#include "isf_check.h"
#include "middleware/cuda_toolkit.h"
#if defined(ISF_ENABLE_TENSORRT)
#include "middleware/cuda_toolkit.h"
#endif
#define APPLE_EXTENSION_SUFFIX ".bundle"
namespace inspire {
@@ -12,7 +20,11 @@ std::mutex Launch::mutex_;
std::shared_ptr<Launch> Launch::instance_ = nullptr;
InspireArchive& Launch::getMArchive() {
return m_archive_;
std::lock_guard<std::mutex> lock(mutex_);
if (!m_archive_) {
throw std::runtime_error("Archive not initialized");
}
return *m_archive_;
}
std::shared_ptr<Launch> Launch::GetInstance() {
@@ -24,21 +36,81 @@ std::shared_ptr<Launch> Launch::GetInstance() {
}
int32_t Launch::Load(const std::string& path) {
std::lock_guard<std::mutex> lock(mutex_);
#if defined(ISF_ENABLE_TENSORRT)
int32_t support_cuda;
auto ret = CheckCudaUsability(&support_cuda);
if (ret != HSUCCEED) {
INSPIRE_LOGE("An error occurred while checking CUDA device support. Please ensure that your environment supports CUDA!");
return ret;
}
if (!support_cuda) {
INSPIRE_LOGE("Your environment does not support CUDA! Please ensure that your environment supports CUDA!");
return HERR_DEVICE_CUDA_NOT_SUPPORT;
}
#endif
INSPIREFACE_CHECK_MSG(os::IsExists(path), "The package path does not exist because the launch failed.");
#if defined(ISF_ENABLE_APPLE_EXTENSION)
BuildAppleExtensionPath(path);
#endif
if (!m_load_) {
m_archive_.ReLoad(path);
if (m_archive_.QueryStatus() == SARC_SUCCESS) {
m_load_ = true;
return HSUCCEED;
} else {
try {
m_archive_ = std::make_unique<InspireArchive>();
m_archive_->ReLoad(path);
if (m_archive_->QueryStatus() == SARC_SUCCESS) {
m_load_ = true;
INSPIRE_LOGI("Successfully loaded resources");
return HSUCCEED;
} else {
m_archive_.reset();
INSPIRE_LOGE("Failed to load resources");
return HERR_ARCHIVE_LOAD_MODEL_FAILURE;
}
} catch (const std::exception& e) {
m_archive_.reset();
INSPIRE_LOGE("Exception during resource loading: %s", e.what());
return HERR_ARCHIVE_LOAD_MODEL_FAILURE;
}
} else {
INSPIRE_LOGW(
"There is no need to call launch more than once, as subsequent calls will not affect the initialization.");
INSPIRE_LOGW("There is no need to call launch more than once, as subsequent calls will not affect the initialization.");
return HSUCCEED;
}
}
int32_t Launch::Reload(const std::string& path) {
std::lock_guard<std::mutex> lock(mutex_);
INSPIREFACE_CHECK_MSG(os::IsExists(path), "The package path does not exist because the launch failed.");
#if defined(ISF_ENABLE_APPLE_EXTENSION)
BuildAppleExtensionPath(path);
#endif
try {
// Clean up existing archive if it exists
if (m_archive_) {
m_archive_.reset();
m_load_ = false;
}
// Create and load new archive
m_archive_ = std::make_unique<InspireArchive>();
m_archive_->ReLoad(path);
if (m_archive_->QueryStatus() == SARC_SUCCESS) {
m_load_ = true;
INSPIRE_LOGI("Successfully reloaded resources");
return HSUCCEED;
} else {
m_archive_.reset();
INSPIRE_LOGE("Failed to reload resources");
return HERR_ARCHIVE_LOAD_MODEL_FAILURE;
}
} catch (const std::exception& e) {
m_archive_.reset();
INSPIRE_LOGE("Exception during resource reloading: %s", e.what());
return HERR_ARCHIVE_LOAD_MODEL_FAILURE;
}
}
bool Launch::isMLoad() const {
return m_load_;
}
@@ -46,8 +118,7 @@ bool Launch::isMLoad() const {
void Launch::Unload() {
std::lock_guard<std::mutex> lock(mutex_);
if (m_load_) {
// Assuming InspireArchive has a method to clear its resources
m_archive_.Release();
m_archive_.reset();
m_load_ = false;
INSPIRE_LOGI("All resources have been successfully unloaded and system is reset.");
} else {
@@ -55,4 +126,54 @@ void Launch::Unload() {
}
}
void Launch::SetRockchipDmaHeapPath(const std::string& path) {
m_rockchip_dma_heap_path_ = path;
}
std::string Launch::GetRockchipDmaHeapPath() const {
return m_rockchip_dma_heap_path_;
}
void Launch::ConfigurationExtensionPath(const std::string& path) {
#if defined(ISF_ENABLE_APPLE_EXTENSION)
INSPIREFACE_CHECK_MSG(os::IsDir(path), "The apple extension path is not a directory, please check.");
#endif
INSPIREFACE_CHECK_MSG(os::IsExists(path), "The extension path is not exists, please check.");
m_extension_path_ = path;
}
std::string Launch::GetExtensionPath() const {
return m_extension_path_;
}
void Launch::SetGlobalCoreMLInferenceMode(InferenceWrapper::SpecialBackend mode) {
m_global_coreml_inference_mode_ = mode;
if (m_global_coreml_inference_mode_ == InferenceWrapper::COREML_CPU) {
INSPIRE_LOGW("Global CoreML Compute Units set to CPU Only.");
} else if (m_global_coreml_inference_mode_ == InferenceWrapper::COREML_GPU) {
INSPIRE_LOGW("Global CoreML Compute Units set to CPU and GPU.");
} else if (m_global_coreml_inference_mode_ == InferenceWrapper::COREML_ANE) {
INSPIRE_LOGW("Global CoreML Compute Units set to Auto Switch (ANE, GPU, CPU).");
}
}
InferenceWrapper::SpecialBackend Launch::GetGlobalCoreMLInferenceMode() const {
return m_global_coreml_inference_mode_;
}
void Launch::BuildAppleExtensionPath(const std::string& resource_path) {
std::string basename = os::Basename(resource_path);
m_extension_path_ = os::PathJoin(os::Dirname(resource_path), basename + APPLE_EXTENSION_SUFFIX);
INSPIREFACE_CHECK_MSG(os::IsExists(m_extension_path_), "The apple extension path is not exists, please check.");
INSPIREFACE_CHECK_MSG(os::IsDir(m_extension_path_), "The apple extension path is not a directory, please check.");
}
void Launch::SetCudaDeviceId(int32_t device_id) {
m_cuda_device_id_ = device_id;
}
int32_t Launch::GetCudaDeviceId() const {
return m_cuda_device_id_;
}
} // namespace inspire

77
cpp-package/inspireface/cpp/inspireface/Initialization_module/launch.h
View File

@@ -1,9 +1,17 @@
// Created by tunm on 2024/04/17.
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef INSPIREFACE_LAUNCH_H
#define INSPIREFACE_LAUNCH_H
#include "middleware/model_archive/inspire_archive.h"
#if defined(ISF_ENABLE_RGA)
#include "middleware/nexus_processor/rga/dma_alloc.h"
#endif
#include <mutex>
#include "middleware/inference_wrapper/inference_wrapper.h"
#include "middleware/system.h"
#ifndef INSPIRE_API
#define INSPIRE_API
@@ -25,7 +33,11 @@ public:
// Loads the necessary resources from a specified path.
// Returns an integer status code: 0 on success, non-zero on failure.
int32_t Load(const std::string &path);
int32_t Load(const std::string& path);
// Reloads the resources from a specified path.
// Returns an integer status code: 0 on success, non-zero on failure.
int32_t Reload(const std::string& path);
// Provides access to the loaded InspireArchive instance.
InspireArchive& getMArchive();
@@ -36,17 +48,62 @@ public:
// Unloads the resources and resets the system to its initial state.
void Unload();
// Set the rockchip dma heap path
void SetRockchipDmaHeapPath(const std::string& path);
// Get the rockchip dma heap path
std::string GetRockchipDmaHeapPath() const;
// Set the extension path
void ConfigurationExtensionPath(const std::string& path);
// Get the extension path
std::string GetExtensionPath() const;
// Set the global coreml inference mode
void SetGlobalCoreMLInferenceMode(InferenceWrapper::SpecialBackend mode);
// Get the global coreml inference mode
InferenceWrapper::SpecialBackend GetGlobalCoreMLInferenceMode() const;
// Build the extension path
void BuildAppleExtensionPath(const std::string& resource_path);
// Set the cuda device id
void SetCudaDeviceId(int32_t device_id);
// Get the cuda device id
int32_t GetCudaDeviceId() const;
private:
Launch() : m_load_(false) {} ///< Private constructor for the singleton pattern.
// Parameters
std::string m_rockchip_dma_heap_path_;
static std::mutex mutex_; ///< Mutex for synchronizing access to the singleton instance.
static std::shared_ptr<Launch> instance_; ///< The singleton instance of Launch.
// Constructor
Launch() : m_load_(false), m_archive_(nullptr) {
#if defined(ISF_ENABLE_RGA)
#if defined(ISF_RKNPU_RV1106)
m_rockchip_dma_heap_path_ = RV1106_CMA_HEAP_PATH;
#else
m_rockchip_dma_heap_path_ = DMA_HEAP_DMA32_UNCACHE_PATCH;
#endif
INSPIRE_LOGW("Rockchip dma heap configured path: %s", m_rockchip_dma_heap_path_.c_str());
#endif
} ///< Private constructor for the singleton pattern.
InspireArchive m_archive_; ///< The archive containing all necessary resources.
bool m_load_; ///< Flag indicating whether the resources have been successfully loaded.
static std::mutex mutex_; ///< Mutex for synchronizing access to the singleton instance.
static std::shared_ptr<Launch> instance_; ///< The singleton instance of Launch.
std::string m_extension_path_;
std::unique_ptr<InspireArchive> m_archive_; ///< The archive containing all necessary resources.
bool m_load_; ///< Flag indicating whether the resources have been successfully loaded.
int32_t m_cuda_device_id_{0};
InferenceWrapper::SpecialBackend m_global_coreml_inference_mode_{InferenceWrapper::COREML_ANE}; ///< The global coreml inference mode
};
} // namespace inspire
} // namespace inspire
#endif //INSPIREFACE_LAUNCH_H
#endif // INSPIREFACE_LAUNCH_H

57
cpp-package/inspireface/cpp/inspireface/Initialization_module/resource_manage.h
View File

@@ -1,4 +1,7 @@
// Created by tunm on 2024/07/16.
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef INSPIREFACE_RESOURCE_MANAGE_H
#define INSPIREFACE_RESOURCE_MANAGE_H
@@ -8,6 +11,7 @@
#include <memory>
#include <iomanip> // For std::setw and std::left
#include <vector>
#include "log.h"
#ifndef INSPIRE_API
#define INSPIRE_API
#endif
@@ -30,6 +34,7 @@ private:
// Use hash tables to store session and image stream handles
std::unordered_map<long, bool> sessionMap;
std::unordered_map<long, bool> streamMap;
std::unordered_map<long, bool> imageBitmapMap;
// The private constructor guarantees singletons
ResourceManager() {}
@@ -84,6 +89,23 @@ public:
// released
}
// Create and record image bitmaps
void createImageBitmap(long handle) {
std::lock_guard<std::mutex> lock(mutex);
imageBitmapMap[handle] = false; // false indicates that it is not released
}
// Release image bitmap
bool releaseImageBitmap(long handle) {
std::lock_guard<std::mutex> lock(mutex);
auto it = imageBitmapMap.find(handle);
if (it != imageBitmapMap.end() && !it->second) {
it->second = true; // Mark as released
return true;
}
return false; // Release failed, possibly because the handle could not be found or was released
}
// Gets a list of unreleased session handles
std::vector<long> getUnreleasedSessions() {
std::lock_guard<std::mutex> lock(mutex);
@@ -108,11 +130,22 @@ public:
return unreleasedStreams;
}
// Gets a list of unreleased image bitmap handles
std::vector<long> getUnreleasedImageBitmaps() {
std::lock_guard<std::mutex> lock(mutex);
std::vector<long> unreleasedImageBitmaps;
for (const auto& entry : imageBitmapMap) {
if (!entry.second) {
unreleasedImageBitmaps.push_back(entry.first);
}
}
return unreleasedImageBitmaps;
}
// Method to print resource management statistics
void printResourceStatistics() {
std::lock_guard<std::mutex> lock(mutex);
std::cout << std::left << std::setw(15) << "Resource Name" << std::setw(15) << "Total Created" << std::setw(15) << "Total Released"
<< std::setw(15) << "Not Released" << std::endl;
INSPIRE_LOGI("%-15s%-15s%-15s%-15s", "Resource Name", "Total Created", "Total Released", "Not Released");
// Print session statistics
int totalSessionsCreated = sessionMap.size();
@@ -124,8 +157,7 @@ public:
if (!entry.second)
++sessionsNotReleased;
}
std::cout << std::left << std::setw(15) << "Session" << std::setw(15) << totalSessionsCreated << std::setw(15) << totalSessionsReleased
<< std::setw(15) << sessionsNotReleased << std::endl;
INSPIRE_LOGI("%-15s%-15d%-15d%-15d", "Session", totalSessionsCreated, totalSessionsReleased, sessionsNotReleased);
// Print stream statistics
int totalStreamsCreated = streamMap.size();
@@ -137,8 +169,19 @@ public:
if (!entry.second)
++streamsNotReleased;
}
std::cout << std::left << std::setw(15) << "Stream" << std::setw(15) << totalStreamsCreated << std::setw(15) << totalStreamsReleased
<< std::setw(15) << streamsNotReleased << std::endl;
INSPIRE_LOGI("%-15s%-15d%-15d%-15d", "Stream", totalStreamsCreated, totalStreamsReleased, streamsNotReleased);
// Print bitmap statistics
int totalBitmapsCreated = imageBitmapMap.size();
int totalBitmapsReleased = 0;
int bitmapsNotReleased = 0;
for (const auto& entry : imageBitmapMap) {
if (entry.second)
++totalBitmapsReleased;
if (!entry.second)
++bitmapsNotReleased;
}
INSPIRE_LOGI("%-15s%-15d%-15d%-15d", "Bitmap", totalBitmapsCreated, totalBitmapsReleased, bitmapsNotReleased);
}
};

657
cpp-package/inspireface/cpp/inspireface/c_api/inspireface.cc
View File

@@ -1,14 +1,20 @@
//
// Created by tunm on 2023/10/3.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "inspireface.h"
#include "intypedef.h"
#include "inspireface_internal.h"
#include "information.h"
#include "feature_hub/feature_hub.h"
#include "Initialization_module/launch.h"
#include "Initialization_module/resource_manage.h"
#include "feature_hub/feature_hub_db.h"
#include "initialization_module/launch.h"
#include "initialization_module/resource_manage.h"
#include "recognition_module/similarity_converter.h"
#include "middleware/inference_wrapper/inference_wrapper.h"
#if defined(ISF_ENABLE_TENSORRT)
#include "middleware/cuda_toolkit.h"
#endif
using namespace inspire;
@@ -20,36 +26,36 @@ HYPER_CAPI_EXPORT extern HResult HFCreateImageStream(PHFImageData data, HFImageS
auto stream = new HF_CameraStream();
switch (data->rotation) {
case HF_CAMERA_ROTATION_90:
stream->impl.SetRotationMode(ROTATION_90);
stream->impl.SetRotationMode(inspirecv::ROTATION_90);
break;
case HF_CAMERA_ROTATION_180:
stream->impl.SetRotationMode(ROTATION_180);
stream->impl.SetRotationMode(inspirecv::ROTATION_180);
break;
case HF_CAMERA_ROTATION_270:
stream->impl.SetRotationMode(ROTATION_270);
stream->impl.SetRotationMode(inspirecv::ROTATION_270);
break;
default:
stream->impl.SetRotationMode(ROTATION_0);
stream->impl.SetRotationMode(inspirecv::ROTATION_0);
break;
}
switch (data->format) {
case HF_STREAM_RGB:
stream->impl.SetDataFormat(RGB);
stream->impl.SetDataFormat(inspirecv::RGB);
break;
case HF_STREAM_BGR:
stream->impl.SetDataFormat(BGR);
stream->impl.SetDataFormat(inspirecv::BGR);
break;
case HF_STREAM_RGBA:
stream->impl.SetDataFormat(RGBA);
stream->impl.SetDataFormat(inspirecv::RGBA);
break;
case HF_STREAM_BGRA:
stream->impl.SetDataFormat(BGRA);
stream->impl.SetDataFormat(inspirecv::BGRA);
break;
case HF_STREAM_YUV_NV12:
stream->impl.SetDataFormat(NV12);
stream->impl.SetDataFormat(inspirecv::NV12);
break;
case HF_STREAM_YUV_NV21:
stream->impl.SetDataFormat(NV21);
stream->impl.SetDataFormat(inspirecv::NV21);
break;
default:
return HERR_INVALID_IMAGE_STREAM_PARAM; // Assume there's a return code for unsupported
@@ -65,6 +71,74 @@ HYPER_CAPI_EXPORT extern HResult HFCreateImageStream(PHFImageData data, HFImageS
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFCreateImageStreamEmpty(HFImageStream *handle) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
auto stream = new HF_CameraStream();
*handle = (HFImageStream)stream;
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetBuffer(HFImageStream handle, HPUInt8 buffer, HInt32 width, HInt32 height) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
((HF_CameraStream *)handle)->impl.SetDataBuffer(buffer, width, height);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetRotation(HFImageStream handle, HFRotation rotation) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
switch (rotation) {
case HF_CAMERA_ROTATION_90:
((HF_CameraStream *)handle)->impl.SetRotationMode(inspirecv::ROTATION_90);
break;
case HF_CAMERA_ROTATION_180:
((HF_CameraStream *)handle)->impl.SetRotationMode(inspirecv::ROTATION_180);
break;
case HF_CAMERA_ROTATION_270:
((HF_CameraStream *)handle)->impl.SetRotationMode(inspirecv::ROTATION_270);
break;
default:
((HF_CameraStream *)handle)->impl.SetRotationMode(inspirecv::ROTATION_0);
break;
}
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetFormat(HFImageStream handle, HFImageFormat format) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
switch (format) {
case HF_STREAM_RGB:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::RGB);
break;
case HF_STREAM_BGR:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::BGR);
break;
case HF_STREAM_RGBA:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::RGBA);
break;
case HF_STREAM_BGRA:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::BGRA);
break;
case HF_STREAM_YUV_NV12:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::NV12);
break;
case HF_STREAM_YUV_NV21:
((HF_CameraStream *)handle)->impl.SetDataFormat(inspirecv::NV21);
break;
default:
return HERR_INVALID_IMAGE_STREAM_PARAM; // Assume there's a return code for unsupported
// formats
}
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFReleaseImageStream(HFImageStream streamHandle) {
if (streamHandle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
@@ -77,6 +151,150 @@ HYPER_CAPI_EXPORT extern HResult HFReleaseImageStream(HFImageStream streamHandle
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmap(PHFImageBitmapData data, HFImageBitmap *handle) {
if (data == nullptr || handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
auto bitmap = new HF_ImageBitmap();
bitmap->impl.Reset(data->width, data->height, data->channels, data->data);
*handle = (HFImageBitmap)bitmap;
// Record the creation of this image bitmap in the ResourceManager
RESOURCE_MANAGE->createImageBitmap((long)*handle);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmapFromFilePath(HPath filePath, HInt32 channels, HFImageBitmap *handle) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
auto image = inspirecv::Image::Create(filePath, channels);
auto bitmap = new HF_ImageBitmap();
bitmap->impl.Reset(image.Width(), image.Height(), image.Channels(), image.Data());
*handle = (HFImageBitmap)bitmap;
// Record the creation of this image bitmap in the ResourceManager
RESOURCE_MANAGE->createImageBitmap((long)*handle);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapCopy(HFImageBitmap handle, HFImageBitmap *copyHandle) {
if (handle == nullptr || copyHandle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
auto bitmap = new HF_ImageBitmap();
bitmap->impl.Reset(((HF_ImageBitmap *)handle)->impl.Width(), ((HF_ImageBitmap *)handle)->impl.Height(),
((HF_ImageBitmap *)handle)->impl.Channels(), ((HF_ImageBitmap *)handle)->impl.Data());
*copyHandle = (HFImageBitmap)bitmap;
// Record the creation of this image bitmap in the ResourceManager
RESOURCE_MANAGE->createImageBitmap((long)*copyHandle);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFReleaseImageBitmap(HFImageBitmap handle) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
// Check and mark this image bitmap as released in the ResourceManager
if (!RESOURCE_MANAGE->releaseImageBitmap((long)handle)) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE; // or other appropriate error code
}
delete (HF_ImageBitmap *)handle;
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFCreateImageStreamFromImageBitmap(HFImageBitmap handle, HFRotation rotation, HFImageStream *streamHandle) {
if (handle == nullptr || streamHandle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
auto stream = new HF_CameraStream();
switch (rotation) {
case HF_CAMERA_ROTATION_90:
stream->impl.SetRotationMode(inspirecv::ROTATION_90);
break;
case HF_CAMERA_ROTATION_180:
stream->impl.SetRotationMode(inspirecv::ROTATION_180);
break;
case HF_CAMERA_ROTATION_270:
stream->impl.SetRotationMode(inspirecv::ROTATION_270);
break;
default:
stream->impl.SetRotationMode(inspirecv::ROTATION_0);
break;
}
stream->impl.SetDataFormat(inspirecv::BGR);
stream->impl.SetDataBuffer(((HF_ImageBitmap *)handle)->impl.Data(), ((HF_ImageBitmap *)handle)->impl.Height(),
((HF_ImageBitmap *)handle)->impl.Width());
*streamHandle = (HFImageStream)stream;
// Record the creation of this stream in the ResourceManager
RESOURCE_MANAGE->createStream((long)*streamHandle);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmapFromImageStreamProcess(HFImageStream streamHandle, HFImageBitmap *handle, int is_rotate,
float scale) {
if (streamHandle == nullptr || handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
auto bitmap = new HF_ImageBitmap();
auto img = ((HF_CameraStream *)streamHandle)->impl.ExecuteImageScaleProcessing(scale, is_rotate);
bitmap->impl.Reset(img.Width(), img.Height(), img.Channels(), img.Data());
*handle = (HFImageBitmap)bitmap;
// Record the creation of this image bitmap in the ResourceManager
RESOURCE_MANAGE->createImageBitmap((long)*handle);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapWriteToFile(HFImageBitmap handle, HPath filePath) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
return ((HF_ImageBitmap *)handle)->impl.Write(filePath);
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawRect(HFImageBitmap handle, HFaceRect rect, HColor color, HInt32 thickness) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
inspirecv::Rect<int> rect_inner(rect.x, rect.y, rect.width, rect.height);
((HF_ImageBitmap *)handle)->impl.DrawRect(rect_inner, {color.r, color.g, color.b}, thickness);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawCircle(HFImageBitmap handle, HPoint2i point, HInt32 radius, HColor color, HInt32 thickness) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
((HF_ImageBitmap *)handle)->impl.DrawCircle({point.x, point.y}, radius, {color.r, color.g, color.b}, thickness);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawCircleF(HFImageBitmap handle, HPoint2f point, HInt32 radius, HColor color, HInt32 thickness) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
((HF_ImageBitmap *)handle)->impl.DrawCircle({(int)point.x, (int)point.y}, radius, {color.r, color.g, color.b}, thickness);
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapGetData(HFImageBitmap handle, PHFImageBitmapData data) {
if (handle == nullptr || data == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
data->width = ((HF_ImageBitmap *)handle)->impl.Width();
data->height = ((HF_ImageBitmap *)handle)->impl.Height();
data->channels = ((HF_ImageBitmap *)handle)->impl.Channels();
data->data = (uint8_t *)((HF_ImageBitmap *)handle)->impl.Data();
return HSUCCEED;
}
HYPER_CAPI_EXPORT extern HResult HFImageBitmapShow(HFImageBitmap handle, HString title, HInt32 delay) {
if (handle == nullptr) {
return HERR_INVALID_IMAGE_BITMAP_HANDLE;
}
((HF_ImageBitmap *)handle)->impl.Show(title, delay);
return HSUCCEED;
}
void HFDeBugImageStreamImShow(HFImageStream streamHandle) {
if (streamHandle == nullptr) {
INSPIRE_LOGE("Handle error");
@@ -86,12 +304,11 @@ void HFDeBugImageStreamImShow(HFImageStream streamHandle) {
INSPIRE_LOGE("Image error");
return;
}
auto image = stream->impl.GetScaledImage(1.0f, true);
auto image = stream->impl.ExecuteImageScaleProcessing(1.0f, true);
#ifdef DISABLE_GUI
cv::imwrite("tmp.jpg", image);
image.Write("tmp.jpg");
#else
cv::imshow("Debug", image);
cv::waitKey(0);
image.Show();
#endif
}
@@ -105,10 +322,10 @@ HResult HFDeBugImageStreamDecodeSave(HFImageStream streamHandle, HPath savePath)
INSPIRE_LOGE("Image error");
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
auto image = stream->impl.GetScaledImage(1.0f, true);
auto ret = cv::imwrite(savePath, image);
auto image = stream->impl.ExecuteImageScaleProcessing(1.0f, true);
auto ret = image.Write(savePath);
if (ret) {
INSPIRE_LOGE("Image saved successfully to %s", savePath);
INSPIRE_LOGI("Image saved successfully to %s", savePath);
return HSUCCEED;
} else {
INSPIRE_LOGE("Failed to save image to %s", savePath);
@@ -138,7 +355,8 @@ HResult HFCreateInspireFaceSession(HFSessionCustomParameter parameter, HFDetectM
param.enable_ir_liveness = parameter.enable_ir_liveness;
param.enable_recognition = parameter.enable_recognition;
param.enable_face_attribute = parameter.enable_face_attribute;
inspire::DetectMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
param.enable_detect_mode_landmark = parameter.enable_detect_mode_landmark;
inspire::DetectModuleMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
if (detectMode == HF_DETECT_MODE_LIGHT_TRACK) {
detMode = inspire::DETECT_MODE_LIGHT_TRACK;
} else if (detectMode == HF_DETECT_MODE_TRACK_BY_DETECTION) {
@@ -183,7 +401,10 @@ HResult HFCreateInspireFaceSessionOptional(HOption customOption, HFDetectMode de
if (customOption & HF_ENABLE_INTERACTION) {
param.enable_interaction_liveness = true;
}
inspire::DetectMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
if (customOption & HF_ENABLE_DETECT_MODE_LANDMARK) {
param.enable_detect_mode_landmark = true;
}
inspire::DetectModuleMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
if (detectMode == HF_DETECT_MODE_LIGHT_TRACK) {
detMode = inspire::DETECT_MODE_LIGHT_TRACK;
} else if (detectMode == HF_DETECT_MODE_TRACK_BY_DETECTION) {
@@ -209,24 +430,107 @@ HResult HFLaunchInspireFace(HPath resourcePath) {
return INSPIRE_LAUNCH->Load(resourcePath);
}
HResult HFReloadInspireFace(HPath resourcePath) {
std::string path(resourcePath);
return INSPIRE_LAUNCH->Reload(resourcePath);
}
HResult HFTerminateInspireFace() {
INSPIRE_LAUNCH->Unload();
return HSUCCEED;
}
HResult HFQueryInspireFaceLaunchStatus(HInt32 *status) {
*status = INSPIRE_LAUNCH->isMLoad();
return HSUCCEED;
}
HResult HFFeatureHubDataDisable() {
return FEATURE_HUB->DisableHub();
return FEATURE_HUB_DB->DisableHub();
}
HResult HFSetExpansiveHardwareRockchipDmaHeapPath(HPath path) {
INSPIRE_LAUNCH->SetRockchipDmaHeapPath(path);
return HSUCCEED;
}
HResult HFQueryExpansiveHardwareRockchipDmaHeapPath(HString path) {
strcpy(path, INSPIRE_LAUNCH->GetRockchipDmaHeapPath().c_str());
return HSUCCEED;
}
HResult HFSetAppleCoreMLInferenceMode(HFAppleCoreMLInferenceMode mode) {
if (mode == HF_APPLE_COREML_INFERENCE_MODE_CPU) {
INSPIRE_LAUNCH->SetGlobalCoreMLInferenceMode(InferenceWrapper::COREML_CPU);
} else if (mode == HF_APPLE_COREML_INFERENCE_MODE_GPU) {
INSPIRE_LAUNCH->SetGlobalCoreMLInferenceMode(InferenceWrapper::COREML_GPU);
} else if (mode == HF_APPLE_COREML_INFERENCE_MODE_ANE) {
INSPIRE_LAUNCH->SetGlobalCoreMLInferenceMode(InferenceWrapper::COREML_ANE);
}
return HSUCCEED;
}
HResult HFSetCudaDeviceId(int32_t device_id) {
INSPIRE_LAUNCH->SetCudaDeviceId(device_id);
return HSUCCEED;
}
HResult HFGetCudaDeviceId(int32_t *device_id) {
*device_id = INSPIRE_LAUNCH->GetCudaDeviceId();
return HSUCCEED;
}
HResult HFPrintCudaDeviceInfo() {
#if defined(ISF_ENABLE_TENSORRT)
return inspire::PrintCudaDeviceInfo();
#else
INSPIRE_LOGW("CUDA is not supported, you need to enable the compile option that supports TensorRT");
return HERR_DEVICE_CUDA_DISABLE;
#endif
}
HResult HFGetNumCudaDevices(int32_t *num_devices) {
#if defined(ISF_ENABLE_TENSORRT)
return inspire::GetCudaDeviceCount(num_devices);
#else
INSPIRE_LOGW("CUDA is not supported, you need to enable the compile option that supports TensorRT");
return HERR_DEVICE_CUDA_DISABLE;
#endif
}
HResult HFCheckCudaDeviceSupport(int32_t *is_support) {
#if defined(ISF_ENABLE_TENSORRT)
return inspire::CheckCudaUsability(is_support);
#else
INSPIRE_LOGW("CUDA is not supported, you need to enable the compile option that supports TensorRT");
return HERR_DEVICE_CUDA_DISABLE;
#endif
}
HResult HFFeatureHubDataEnable(HFFeatureHubConfiguration configuration) {
inspire::DatabaseConfiguration param;
param.db_path = (configuration.dbPath != nullptr) ? std::string(configuration.dbPath) : std::string();
param.enable_use_db = configuration.enablePersistence;
param.feature_block_num = configuration.featureBlockNum;
if (configuration.primaryKeyMode != HF_PK_AUTO_INCREMENT && configuration.primaryKeyMode != HF_PK_MANUAL_INPUT) {
param.primary_key_mode = inspire::PrimaryKeyMode::AUTO_INCREMENT;
} else {
param.primary_key_mode = inspire::PrimaryKeyMode(configuration.primaryKeyMode);
}
if (configuration.persistenceDbPath == nullptr) {
INSPIRE_LOGE("persistenceDbPath is null, use default path");
}
// Add validation for persistenceDbPath
if (configuration.enablePersistence) {
if (configuration.persistenceDbPath == nullptr) {
param.persistence_db_path = std::string("");
} else {
param.persistence_db_path = std::string(configuration.persistenceDbPath);
}
} else {
param.persistence_db_path = std::string(""); // Empty string for in-memory mode
}
param.enable_persistence = configuration.enablePersistence;
param.recognition_threshold = configuration.searchThreshold;
param.search_mode = (SearchMode)configuration.searchMode;
auto ret = FEATURE_HUB->EnableHub(param);
param.search_mode = (inspire::SearchMode)configuration.searchMode;
auto ret = FEATURE_HUB_DB->EnableHub(param);
return ret;
}
@@ -260,7 +564,7 @@ HResult HFSessionSetFaceTrackMode(HFSession session, HFDetectMode detectMode) {
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
inspire::DetectMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
inspire::DetectModuleMode detMode = inspire::DETECT_MODE_ALWAYS_DETECT;
if (detectMode == HF_DETECT_MODE_LIGHT_TRACK) {
detMode = inspire::DETECT_MODE_LIGHT_TRACK;
}
@@ -278,6 +582,39 @@ HResult HFSessionSetFaceDetectThreshold(HFSession session, HFloat threshold) {
return ctx->impl.SetFaceDetectThreshold(threshold);
}
HResult HFSessionSetTrackModeSmoothRatio(HFSession session, HFloat ratio) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
return ctx->impl.SetTrackModeSmoothRatio(ratio);
}
HResult HFSessionSetTrackModeNumSmoothCacheFrame(HFSession session, HInt32 num) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
return ctx->impl.SetTrackModeNumSmoothCacheFrame(num);
}
HResult HFSessionSetTrackModeDetectInterval(HFSession session, HInt32 num) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
return ctx->impl.SetTrackModeDetectInterval(num);
}
HResult HFExecuteFaceTrack(HFSession session, HFImageStream streamHandle, PHFMultipleFaceData results) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
@@ -333,20 +670,67 @@ HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2
data.data = singleFace.data;
HyperFaceData face = {0};
HInt32 ret;
ret = DeserializeHyperFaceData((char *)data.data, data.dataSize, face);
ret = RunDeserializeHyperFaceData((char *)data.data, data.dataSize, face);
if (ret != HSUCCEED) {
return ret;
}
if (face.densityLandmarkEnable == 0) {
INSPIRE_LOGW("To get dense landmarks in always-detect mode, you need to enable HF_ENABLE_DETECT_MODE_LANDMARK");
return HERR_SESS_LANDMARK_NOT_ENABLE;
}
for (size_t i = 0; i < num; i++) {
landmarks[i].x = face.densityLandmark[i].x;
landmarks[i].y = face.densityLandmark[i].y;
}
return HSUCCEED;
}
HResult HFGetFaceFiveKeyPointsFromFaceToken(HFFaceBasicToken singleFace, HPoint2f *landmarks, HInt32 num) {
if (num != 5) {
return HERR_SESS_KEY_POINT_NUM_NOT_MATCH;
}
inspire::FaceBasicData data;
data.dataSize = singleFace.size;
data.data = singleFace.data;
HyperFaceData face = {0};
HInt32 ret;
ret = RunDeserializeHyperFaceData((char *)data.data, data.dataSize, face);
if (ret != HSUCCEED) {
return ret;
}
for (size_t i = 0; i < num; i++) {
landmarks[i].x = face.keyPoints[i].x;
landmarks[i].y = face.keyPoints[i].y;
}
return HSUCCEED;
}
HResult HFSessionSetEnableTrackCostSpend(HFSession session, int value) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
ctx->impl.SetEnableTrackCostSpend(value);
return HSUCCEED;
}
HResult HFSessionPrintTrackCostSpend(HFSession session) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
ctx->impl.PrintTrackCostSpend();
return HSUCCEED;
}
HResult HFFeatureHubFaceSearchThresholdSetting(float threshold) {
FEATURE_HUB->SetRecognitionThreshold(threshold);
FEATURE_HUB_DB->SetRecognitionThreshold(threshold);
return HSUCCEED;
}
@@ -407,6 +791,39 @@ HResult HFFaceFeatureExtractCpy(HFSession session, HFImageStream streamHandle, H
return ret;
}
HResult HFFaceGetFaceAlignmentImage(HFSession session, HFImageStream streamHandle, HFFaceBasicToken singleFace, HFImageBitmap *handle) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
if (streamHandle == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession *)session;
if (ctx == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
HF_CameraStream *stream = (HF_CameraStream *)streamHandle;
if (stream == nullptr) {
return HERR_INVALID_IMAGE_STREAM_HANDLE;
}
if (singleFace.data == nullptr || singleFace.size <= 0) {
return HERR_INVALID_FACE_TOKEN;
}
inspire::FaceBasicData data;
data.dataSize = singleFace.size;
data.data = singleFace.data;
auto bitmap = new HF_ImageBitmap();
auto ret = ctx->impl.FaceGetFaceAlignmentImage(stream->impl, data, bitmap->impl);
if (ret != HSUCCEED) {
delete bitmap;
return ret;
}
*handle = bitmap;
// Record the creation of this image bitmap in the ResourceManager
RESOURCE_MANAGE->createImageBitmap((long)*handle);
return HSUCCEED;
}
HResult HFFaceComparison(HFFaceFeature feature1, HFFaceFeature feature2, HPFloat result) {
if (feature1.data == nullptr || feature2.data == nullptr) {
return HERR_INVALID_FACE_FEATURE;
@@ -417,19 +834,62 @@ HResult HFFaceComparison(HFFaceFeature feature1, HFFaceFeature feature2, HPFloat
}
*result = 0.0f;
float res = -1.0f;
auto ret = FEATURE_HUB->CosineSimilarity(feature1.data, feature2.data, feature1.size, res);
auto ret = FEATURE_HUB_DB->CosineSimilarity(feature1.data, feature2.data, feature1.size, res);
*result = res;
return ret;
}
HResult HFGetRecommendedCosineThreshold(HPFloat threshold) {
if (!INSPIRE_LAUNCH->isMLoad()) {
INSPIRE_LOGW("Inspireface is not launched, using default threshold 0.48");
}
*threshold = SIMILARITY_CONVERTER_GET_RECOMMENDED_COSINE_THRESHOLD();
return HSUCCEED;
}
HResult HFCosineSimilarityConvertToPercentage(HFloat similarity, HPFloat result) {
if (!INSPIRE_LAUNCH->isMLoad()) {
INSPIRE_LOGW("Inspireface is not launched.");
}
*result = SIMILARITY_CONVERTER_RUN(similarity);
return HSUCCEED;
}
HResult HFUpdateCosineSimilarityConverter(HFSimilarityConverterConfig config) {
if (!INSPIRE_LAUNCH->isMLoad()) {
INSPIRE_LOGW("Inspireface is not launched.");
}
inspire::SimilarityConverterConfig cfg;
cfg.threshold = config.threshold;
cfg.middleScore = config.middleScore;
cfg.steepness = config.steepness;
cfg.outputMin = config.outputMin;
cfg.outputMax = config.outputMax;
SIMILARITY_CONVERTER_UPDATE_CONFIG(cfg);
return HSUCCEED;
}
HResult HFGetCosineSimilarityConverter(PHFSimilarityConverterConfig config) {
if (!INSPIRE_LAUNCH->isMLoad()) {
INSPIRE_LOGW("Inspireface is not launched.");
}
inspire::SimilarityConverterConfig cfg = SIMILARITY_CONVERTER_GET_CONFIG();
config->threshold = cfg.threshold;
config->middleScore = cfg.middleScore;
config->steepness = cfg.steepness;
config->outputMin = cfg.outputMin;
config->outputMax = cfg.outputMax;
return HSUCCEED;
}
HResult HFGetFeatureLength(HPInt32 num) {
*num = FEATURE_HUB->GetFeatureNum();
*num = 512;
return HSUCCEED;
}
HResult HFFeatureHubInsertFeature(HFFaceFeatureIdentity featureIdentity) {
HResult HFFeatureHubInsertFeature(HFFaceFeatureIdentity featureIdentity, HPFaceId allocId) {
if (featureIdentity.feature->data == nullptr) {
return HERR_INVALID_FACE_FEATURE;
}
@@ -438,8 +898,7 @@ HResult HFFeatureHubInsertFeature(HFFaceFeatureIdentity featureIdentity) {
for (int i = 0; i < featureIdentity.feature->size; ++i) {
feat.push_back(featureIdentity.feature->data[i]);
}
std::string tag(featureIdentity.tag);
HInt32 ret = FEATURE_HUB->FaceFeatureInsertFromCustomId(feat, tag, featureIdentity.customId);
HInt32 ret = FEATURE_HUB_DB->FaceFeatureInsert(feat, featureIdentity.id, *allocId);
return ret;
}
@@ -453,14 +912,16 @@ HResult HFFeatureHubFaceSearch(HFFaceFeature searchFeature, HPFloat confidence,
for (int i = 0; i < searchFeature.size; ++i) {
feat.push_back(searchFeature.data[i]);
}
inspire::SearchResult result;
HInt32 ret = FEATURE_HUB->SearchFaceFeature(feat, result);
mostSimilar->feature = (HFFaceFeature *)FEATURE_HUB->GetFaceFeaturePtrCache().get();
mostSimilar->feature->data = (HFloat *)FEATURE_HUB->GetSearchFaceFeatureCache().data();
mostSimilar->feature->size = FEATURE_HUB->GetSearchFaceFeatureCache().size();
mostSimilar->tag = FEATURE_HUB->GetStringCache();
mostSimilar->customId = result.customId;
*confidence = result.score;
*confidence = -1.0f;
inspire::FaceSearchResult result;
HInt32 ret = FEATURE_HUB_DB->SearchFaceFeature(feat, result);
mostSimilar->feature = (HFFaceFeature *)FEATURE_HUB_DB->GetFaceFeaturePtrCache().get();
mostSimilar->feature->data = (HFloat *)FEATURE_HUB_DB->GetSearchFaceFeatureCache().data();
mostSimilar->feature->size = FEATURE_HUB_DB->GetSearchFaceFeatureCache().size();
mostSimilar->id = result.id;
if (mostSimilar->id != -1) {
*confidence = result.similarity;
}
return ret;
}
@@ -474,18 +935,18 @@ HResult HFFeatureHubFaceSearchTopK(HFFaceFeature searchFeature, HInt32 topK, PHF
for (int i = 0; i < searchFeature.size; ++i) {
feat.push_back(searchFeature.data[i]);
}
HInt32 ret = FEATURE_HUB->SearchFaceFeatureTopK(feat, topK);
HInt32 ret = FEATURE_HUB_DB->SearchFaceFeatureTopKCache(feat, topK);
if (ret == HSUCCEED) {
results->size = FEATURE_HUB->GetTopKConfidence().size();
results->confidence = FEATURE_HUB->GetTopKConfidence().data();
results->customIds = FEATURE_HUB->GetTopKCustomIdsCache().data();
results->size = FEATURE_HUB_DB->GetTopKConfidence().size();
results->confidence = FEATURE_HUB_DB->GetTopKConfidence().data();
results->ids = FEATURE_HUB_DB->GetTopKCustomIdsCache().data();
}
return ret;
}
HResult HFFeatureHubFaceRemove(HInt32 customId) {
auto ret = FEATURE_HUB->FaceFeatureRemoveFromCustomId(customId);
HResult HFFeatureHubFaceRemove(HFaceId id) {
auto ret = FEATURE_HUB_DB->FaceFeatureRemove(id);
return ret;
}
@@ -498,23 +959,21 @@ HResult HFFeatureHubFaceUpdate(HFFaceFeatureIdentity featureIdentity) {
for (int i = 0; i < featureIdentity.feature->size; ++i) {
feat.push_back(featureIdentity.feature->data[i]);
}
std::string tag(featureIdentity.tag);
auto ret = FEATURE_HUB->FaceFeatureUpdateFromCustomId(feat, tag, featureIdentity.customId);
auto ret = FEATURE_HUB_DB->FaceFeatureUpdate(feat, featureIdentity.id);
return ret;
}
HResult HFFeatureHubGetFaceIdentity(HInt32 customId, PHFFaceFeatureIdentity identity) {
auto ret = FEATURE_HUB->GetFaceFeatureFromCustomId(customId);
HResult HFFeatureHubGetFaceIdentity(HFaceId id, PHFFaceFeatureIdentity identity) {
auto ret = FEATURE_HUB_DB->GetFaceFeature(id);
if (ret == HSUCCEED) {
identity->tag = FEATURE_HUB->GetStringCache();
identity->customId = customId;
identity->feature = (HFFaceFeature *)FEATURE_HUB->GetFaceFeaturePtrCache().get();
identity->feature->data = (HFloat *)FEATURE_HUB->GetFaceFeaturePtrCache()->data;
identity->feature->size = FEATURE_HUB->GetFaceFeaturePtrCache()->dataSize;
identity->id = id;
identity->feature = (HFFaceFeature *)FEATURE_HUB_DB->GetFaceFeaturePtrCache().get();
identity->feature->data = (HFloat *)FEATURE_HUB_DB->GetFaceFeaturePtrCache()->data;
identity->feature->size = FEATURE_HUB_DB->GetFaceFeaturePtrCache()->dataSize;
} else {
identity->customId = -1;
identity->id = -1;
}
return ret;
@@ -551,13 +1010,14 @@ HResult HFMultipleFacePipelineProcess(HFSession session, HFImageStream streamHan
param.enable_ir_liveness = parameter.enable_ir_liveness;
param.enable_recognition = parameter.enable_recognition;
param.enable_face_attribute = parameter.enable_face_attribute;
param.enable_detect_mode_landmark = parameter.enable_detect_mode_landmark;
HResult ret;
std::vector<inspire::HyperFaceData> data;
data.resize(faces->detectedNum);
for (int i = 0; i < faces->detectedNum; ++i) {
auto &face = data[i];
ret = DeserializeHyperFaceData((char *)faces->tokens[i].data, faces->tokens[i].size, face);
ret = RunDeserializeHyperFaceData((char *)faces->tokens[i].data, faces->tokens[i].size, face);
if (ret != HSUCCEED) {
return HERR_INVALID_FACE_TOKEN;
}
@@ -612,13 +1072,16 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
if (customOption & HF_ENABLE_INTERACTION) {
param.enable_interaction_liveness = true;
}
if (customOption & HF_ENABLE_DETECT_MODE_LANDMARK) {
param.enable_detect_mode_landmark = true;
}
HResult ret;
std::vector<inspire::HyperFaceData> data;
data.resize(faces->detectedNum);
for (int i = 0; i < faces->detectedNum; ++i) {
auto &face = data[i];
ret = DeserializeHyperFaceData((char *)faces->tokens[i].data, faces->tokens[i].size, face);
ret = RunDeserializeHyperFaceData((char *)faces->tokens[i].data, faces->tokens[i].size, face);
if (ret != HSUCCEED) {
return HERR_INVALID_FACE_TOKEN;
}
@@ -687,12 +1150,12 @@ HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFlo
data.dataSize = singleFace.size;
data.data = singleFace.data;
auto ret = inspire::FaceContext::FaceQualityDetect(data, *confidence);
auto ret = inspire::FaceSession::FaceQualityDetect(data, *confidence);
return ret;
}
HResult HFGetFaceIntereactionStateResult(HFSession session, PHFFaceIntereactionState result) {
HResult HFGetFaceInteractionStateResult(HFSession session, PHFFaceInteractionState result) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
@@ -707,7 +1170,7 @@ HResult HFGetFaceIntereactionStateResult(HFSession session, PHFFaceIntereactionS
return HSUCCEED;
}
HResult HFGetFaceIntereactionActionsResult(HFSession session, PHFFaceIntereactionsActions actions) {
HResult HFGetFaceInteractionActionsResult(HFSession session, PHFFaceInteractionsActions actions) {
if (session == nullptr) {
return HERR_INVALID_CONTEXT_HANDLE;
}
@@ -719,7 +1182,7 @@ HResult HFGetFaceIntereactionActionsResult(HFSession session, PHFFaceIntereactio
actions->normal = (HInt32 *)ctx->impl.GetFaceNormalAactionsResultCache().data();
actions->blink = (HInt32 *)ctx->impl.GetFaceBlinkAactionsResultCache().data();
actions->shake = (HInt32 *)ctx->impl.GetFaceShakeAactionsResultCache().data();
actions->headRiase = (HInt32 *)ctx->impl.GetFaceRaiseHeadAactionsResultCache().data();
actions->headRaise = (HInt32 *)ctx->impl.GetFaceRaiseHeadAactionsResultCache().data();
actions->jawOpen = (HInt32 *)ctx->impl.GetFaceJawOpenAactionsResultCache().data();
return HSUCCEED;
@@ -743,12 +1206,22 @@ HResult HFGetFaceAttributeResult(HFSession session, PHFFaceAttributeResult resul
}
HResult HFFeatureHubGetFaceCount(HInt32 *count) {
*count = FEATURE_HUB->GetFaceFeatureCount();
*count = FEATURE_HUB_DB->GetFaceFeatureCount();
return HSUCCEED;
}
HResult HFFeatureHubViewDBTable() {
return FEATURE_HUB->ViewDBTable();
FEATURE_HUB_DB->ViewDBTable();
return HSUCCEED;
}
HResult HFFeatureHubGetExistingIds(PHFFeatureHubExistingIds ids) {
auto ret = FEATURE_HUB_DB->GetAllIds();
if (ret == HSUCCEED) {
ids->size = FEATURE_HUB_DB->GetExistingIds().size();
ids->ids = FEATURE_HUB_DB->GetExistingIds().data();
}
return ret;
}
HResult HFQueryInspireFaceVersion(PHFInspireFaceVersion version) {
@@ -759,6 +1232,11 @@ HResult HFQueryInspireFaceVersion(PHFInspireFaceVersion version) {
return HSUCCEED;
}
HResult HFQueryInspireFaceExtendedInformation(PHFInspireFaceExtendedInformation information) {
strncpy(information->information, INSPIRE_FACE_EXTENDED_INFORMATION, strlen(INSPIRE_FACE_EXTENDED_INFORMATION));
return HSUCCEED;
}
HResult HFSetLogLevel(HFLogLevel level) {
INSPIRE_SET_LOG_LEVEL(LogLevel(level));
return HSUCCEED;
@@ -766,6 +1244,41 @@ HResult HFSetLogLevel(HFLogLevel level) {
HResult HFLogDisable() {
INSPIRE_SET_LOG_LEVEL(inspire::ISF_LOG_NONE);
return HSUCCEED;
}
HResult HFLogPrint(HFLogLevel level, HFormat format, ...) {
inspire::LogLevel logLevel = static_cast<inspire::LogLevel>(level);
if (inspire::LogManager::getInstance()->getLogLevel() == inspire::ISF_LOG_NONE || logLevel < inspire::LogManager::getInstance()->getLogLevel()) {
return HSUCCEED;
}
char buffer[1024];
va_list args;
va_start(args, format);
vsnprintf(buffer, sizeof(buffer), format, args);
va_end(args);
switch (logLevel) {
case inspire::ISF_LOG_DEBUG:
INSPIRE_LOGD("%s", buffer);
break;
case inspire::ISF_LOG_INFO:
INSPIRE_LOGI("%s", buffer);
break;
case inspire::ISF_LOG_WARN:
INSPIRE_LOGW("%s", buffer);
break;
case inspire::ISF_LOG_ERROR:
INSPIRE_LOGE("%s", buffer);
break;
case inspire::ISF_LOG_FATAL:
INSPIRE_LOGF("%s", buffer);
break;
default:
break;
}
return HSUCCEED;
}

496
cpp-package/inspireface/cpp/inspireface/c_api/inspireface.h
View File

@@ -1,9 +1,10 @@
//
// Created by tunm on 2023/10/3.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef HYPERFACEREPO_INSPIREFACE_H
#define HYPERFACEREPO_INSPIREFACE_H
#ifndef INSPIREFACE_H
#define INSPIREFACE_H
#include <stdint.h>
#include "intypedef.h"
@@ -23,15 +24,16 @@
extern "C" {
#endif
#define HF_ENABLE_NONE 0x00000000 ///< Flag to enable no features.
#define HF_ENABLE_FACE_RECOGNITION 0x00000002 ///< Flag to enable face recognition feature.
#define HF_ENABLE_LIVENESS 0x00000004 ///< Flag to enable RGB liveness detection feature.
#define HF_ENABLE_IR_LIVENESS 0x00000008 ///< Flag to enable IR (Infrared) liveness detection feature.
#define HF_ENABLE_MASK_DETECT 0x00000010 ///< Flag to enable mask detection feature.
#define HF_ENABLE_FACE_ATTRIBUTE 0x00000020 ///< Flag to enable face attribute prediction feature.
#define HF_ENABLE_PLACEHOLDER_ 0x00000040 ///< -
#define HF_ENABLE_QUALITY 0x00000080 ///< Flag to enable face quality assessment feature.
#define HF_ENABLE_INTERACTION 0x00000100 ///< Flag to enable interaction feature.
#define HF_ENABLE_NONE 0x00000000 ///< Flag to enable no features.
#define HF_ENABLE_FACE_RECOGNITION 0x00000002 ///< Flag to enable face recognition feature.
#define HF_ENABLE_LIVENESS 0x00000004 ///< Flag to enable RGB liveness detection feature.
#define HF_ENABLE_IR_LIVENESS 0x00000008 ///< Flag to enable IR (Infrared) liveness detection feature.
#define HF_ENABLE_MASK_DETECT 0x00000010 ///< Flag to enable mask detection feature.
#define HF_ENABLE_FACE_ATTRIBUTE 0x00000020 ///< Flag to enable face attribute prediction feature.
#define HF_ENABLE_PLACEHOLDER_ 0x00000040 ///< -
#define HF_ENABLE_QUALITY 0x00000080 ///< Flag to enable face quality assessment feature.
#define HF_ENABLE_INTERACTION 0x00000100 ///< Flag to enable interaction feature.
#define HF_ENABLE_DETECT_MODE_LANDMARK 0x00000200 ///< Flag to enable landmark detection in detection mode
/**
* Camera stream format.
@@ -62,7 +64,7 @@ typedef enum HFRotation {
* Defines the structure for image data stream.
*/
typedef struct HFImageData {
uint8_t *data; ///< Pointer to the image data stream.
HPUInt8 data; ///< Pointer to the image data stream.
HInt32 width; ///< Width of the image.
HInt32 height; ///< Height of the image.
HFImageFormat format; ///< Format of the image, indicating the data stream format to be parsed.
@@ -80,6 +82,45 @@ typedef struct HFImageData {
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageStream(PHFImageData data, HFImageStream *handle);
/**
* @brief Create an empty image stream instance.
*
* This function is used to create an instance of a data buffer stream with the given image data.
*
* @param handle Pointer to the stream handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageStreamEmpty(HFImageStream *handle);
/**
* @brief Set the buffer of the image stream.
*
* @param handle Pointer to the stream handle.
* @param buffer Pointer to the buffer.
* @param width Width of the image.
* @param height Height of the image.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetBuffer(HFImageStream handle, HPUInt8 buffer, HInt32 width, HInt32 height);
/**
* @brief Set the rotation of the image stream.
*
* @param handle Pointer to the stream handle.
* @param rotation Rotation angle of the image.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetRotation(HFImageStream handle, HFRotation rotation);
/**
* @brief Set the format of the image stream.
*
* @param handle Pointer to the stream handle.
* @param format Format of the image.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageStreamSetFormat(HFImageStream handle, HFImageFormat format);
/**
* @brief Release the instantiated DataBuffer object.
*
@@ -90,6 +131,126 @@ HYPER_CAPI_EXPORT extern HResult HFCreateImageStream(PHFImageData data, HFImageS
*/
HYPER_CAPI_EXPORT extern HResult HFReleaseImageStream(HFImageStream streamHandle);
/**
* @brief Struct for image bitmap data.
*/
typedef struct HFImageBitmapData {
uint8_t *data; ///< Pointer to the image data.
HInt32 width; ///< Width of the image.
HInt32 height; ///< Height of the image.
HInt32 channels; ///< Number of channels in the image, only support 3 channels or 1 channel.
} HFImageBitmapData, *PHFImageBitmapData;
/**
* @brief Create a image bitmap from data, default pixel format is BGR.
*
* @param data Pointer to the image bitmap data structure.
* @param handle Pointer to the image bitmap handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmap(PHFImageBitmapData data, HFImageBitmap *handle);
/**
* @brief Create a image bitmap from file path, default pixel format is BGR.
*
* @param filePath The path to the image file.
* @param channels The number of channels in the image, only support 3 channels or 1 channel.
* @param handle Pointer to the image bitmap handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmapFromFilePath(HPath filePath, HInt32 channels, HFImageBitmap *handle);
/**
* @brief Copy an image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @param copyHandle Pointer to the image bitmap handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapCopy(HFImageBitmap handle, HFImageBitmap *copyHandle);
/**
* @brief Release the image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFReleaseImageBitmap(HFImageBitmap handle);
/**
* @brief Create a image stream from image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @param rotation The rotation angle of the image.
* @param streamHandle Pointer to the image stream handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageStreamFromImageBitmap(HFImageBitmap handle, HFRotation rotation, HFImageStream *streamHandle);
/**
* @brief Create a image bitmap from image stream.
*
* @param streamHandle Pointer to the image stream handle.
* @param handle Pointer to the image bitmap handle that will be returned.
* @param is_rotate Whether to rotate the image.
* @param scale The scale of the image.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCreateImageBitmapFromImageStreamProcess(HFImageStream streamHandle, HFImageBitmap *handle, int is_rotate,
float scale);
/**
* @brief Write the image bitmap to a file.
*
* @param handle Pointer to the image bitmap handle.
* @param filePath The path to the image file.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapWriteToFile(HFImageBitmap handle, HPath filePath);
/**
* @brief Draw a rectangle on the image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @param rect The rectangle to be drawn.
* @param color The color of the rectangle.
* @param thickness The thickness of the rectangle.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawRect(HFImageBitmap handle, HFaceRect rect, HColor color, HInt32 thickness);
/**
* @brief Draw a circle on the image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @param point The center point of the circle.
* @param radius The radius of the circle.
* @param color The color of the circle.
* @param thickness The thickness of the circle.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawCircleF(HFImageBitmap handle, HPoint2f point, HInt32 radius, HColor color, HInt32 thickness);
HYPER_CAPI_EXPORT extern HResult HFImageBitmapDrawCircle(HFImageBitmap handle, HPoint2i point, HInt32 radius, HColor color, HInt32 thickness);
/**
* @brief Get the data of the image bitmap.
*
* @param handle Pointer to the image bitmap handle.
* @param data Pointer to the image bitmap data structure.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapGetData(HFImageBitmap handle, PHFImageBitmapData data);
/**
* @brief Show the image bitmap.
*
* @param handle Pointer to the image bitmap handle, must rely on opencv's gui functionality
* @param title The title of the image.
* @param delay The delay time of the image.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFImageBitmapShow(HFImageBitmap handle, HString title, HInt32 delay);
/************************************************************************
* Resource Function
************************************************************************/
@@ -104,6 +265,14 @@ HYPER_CAPI_EXPORT extern HResult HFReleaseImageStream(HFImageStream streamHandle
* */
HYPER_CAPI_EXPORT extern HResult HFLaunchInspireFace(HPath resourcePath);
/**
* @brief Reload InspireFace SDK
* Reload the InspireFace SDK, releasing all allocated resources.
* @param resourcePath Initializes the path to the resource file that needs to be loaded
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFReloadInspireFace(HPath resourcePath);
/**
* @brief Terminate InspireFace SDK
* Terminate the InspireFace SDK, releasing all allocated resources.
@@ -112,8 +281,92 @@ HYPER_CAPI_EXPORT extern HResult HFLaunchInspireFace(HPath resourcePath);
* */
HYPER_CAPI_EXPORT extern HResult HFTerminateInspireFace();
/**
* @brief Query InspireFace SDK launch status
* Query the launch status of the InspireFace SDK.
* @param status Pointer to the status variable that will be returned.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFQueryInspireFaceLaunchStatus(HInt32 *status);
/************************************************************************
* FaceContext
* Extended Interface Based on Third-party Hardware Devices
*
* According to different manufacturers' devices, manufacturers typically perform deep customization and optimization, such as neural network
* inference computation, geometric image acceleration computation, and deeply customized device interfaces, etc. These types of functionalities are
* usually difficult to abstract, so they are placed in extension module APIs, involving hybrid computing, heterogeneous computing, multi-device
* computing, and other features.
************************************************************************/
/**
* @brief Set the rockchip dma heap path
* By default, we have already configured the DMA Heap address used by RGA on RK devices.
* If you wish to customize this address, you can modify it through this API.
* @param path The path to the rockchip dma heap
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFSetExpansiveHardwareRockchipDmaHeapPath(HPath path);
/**
* @brief Query the rockchip dma heap path
* @param path Pointer to a pre-allocated character array that will store the returned path.
* The array should be at least 256 bytes in size.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFQueryExpansiveHardwareRockchipDmaHeapPath(HString path);
/**
* @brief Enum for Apple CoreML inference mode.
*/
typedef enum HFAppleCoreMLInferenceMode {
HF_APPLE_COREML_INFERENCE_MODE_CPU = 0, ///< CPU Only.
HF_APPLE_COREML_INFERENCE_MODE_GPU = 1, ///< GPU first.
HF_APPLE_COREML_INFERENCE_MODE_ANE = 2, ///< Automatic selection, ANE first.
} HFAppleCoreMLInferenceMode;
/**
* @brief Set the Apple CoreML inference mode, must be called before HFCreateInspireFaceSession.
* @param mode The inference mode to be set.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFSetAppleCoreMLInferenceMode(HFAppleCoreMLInferenceMode mode);
/**
* @brief Set the CUDA device id, must be called before HFCreateInspireFaceSession.
* @param device_id The device id to be set.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFSetCudaDeviceId(int32_t device_id);
/**
* @brief Get the CUDA device id, must be called after HFCreateInspireFaceSession.
* @param device_id Pointer to the device id to be returned.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFGetCudaDeviceId(int32_t *device_id);
/**
* @brief Print the CUDA device information.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFPrintCudaDeviceInfo();
/**
* @brief Get the number of CUDA devices.
* @param num_devices Pointer to the number of CUDA devices to be returned.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFGetNumCudaDevices(int32_t *num_devices);
/**
* @brief Check if the CUDA device is supported.
* @param support The support flag to be checked.
* @return HResult indicating the success or failure of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFCheckCudaDeviceSupport(int32_t *is_support);
/************************************************************************
* FaceSession
************************************************************************/
/**
@@ -131,6 +384,7 @@ typedef struct HFSessionCustomParameter {
HInt32 enable_face_quality; ///< Enable face quality detection feature.
HInt32 enable_face_attribute; ///< Enable face attribute prediction feature.
HInt32 enable_interaction_liveness; ///< Enable interaction for liveness detection feature.
HInt32 enable_detect_mode_landmark; ///< Enable landmark detection in detection mode
} HFSessionCustomParameter, *PHFSessionCustomParameter;
/**
@@ -253,6 +507,33 @@ HYPER_CAPI_EXPORT extern HResult HFSessionSetFilterMinimumFacePixelSize(HFSessio
*/
HYPER_CAPI_EXPORT extern HResult HFSessionSetFaceDetectThreshold(HFSession session, HFloat threshold);
/**
* @brief Set the track mode smooth ratio in the session. default value is 0.05
*
* @param session Handle to the session.
* @param ratio The smooth ratio value.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFSessionSetTrackModeSmoothRatio(HFSession session, HFloat ratio);
/**
* @brief Set the track mode num smooth cache frame in the session. default value is 5
*
* @param session Handle to the session.
* @param num The num smooth cache frame value.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFSessionSetTrackModeNumSmoothCacheFrame(HFSession session, HInt32 num);
/**
* @brief Set the track model detect interval in the session. default value is 20
*
* @param session Handle to the session.
* @param num The detect interval value.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFSessionSetTrackModeDetectInterval(HFSession session, HInt32 num);
/**
* @brief Run face tracking in the session.
*
@@ -311,6 +592,29 @@ HYPER_CAPI_EXPORT extern HResult HFGetNumOfFaceDenseLandmark(HPInt32 num);
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f *landmarks, HInt32 num);
/**
* @brief Get the five key points from the face token.
* @param singleFace Basic token representing a single face.
* @param landmarks Pre-allocated memory address of the array for 2D floating-point coordinates.
* @param num Number of landmark points
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceFiveKeyPointsFromFaceToken(HFFaceBasicToken singleFace, HPoint2f *landmarks, HInt32 num);
/**
* @brief Set the enable cost spend
* @param value The enable cost spend value
* @return int32_t Status code of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFSessionSetEnableTrackCostSpend(HFSession session, int value);
/**
* @brief Print the cost spend
* @param session The session handle
* @return int32_t Status code of the operation.
* */
HYPER_CAPI_EXPORT extern HResult HFSessionPrintTrackCostSpend(HFSession session);
/************************************************************************
* Face Recognition
************************************************************************/
@@ -348,6 +652,17 @@ HYPER_CAPI_EXPORT extern HResult HFFaceFeatureExtract(HFSession session, HFImage
*/
HYPER_CAPI_EXPORT extern HResult HFFaceFeatureExtractCpy(HFSession session, HFImageStream streamHandle, HFFaceBasicToken singleFace, HPFloat feature);
/**
* @brief Get the face alignment image.
* @param session Handle to the session.
* @param streamHandle Handle to the data buffer representing the camera stream component.
* @param singleFace Basic token representing a single face.
* @param handle Pointer to the handle that will be returned.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFaceGetFaceAlignmentImage(HFSession session, HFImageStream streamHandle, HFFaceBasicToken singleFace,
HFImageBitmap *handle);
/************************************************************************
* Feature Hub
************************************************************************/
@@ -361,19 +676,25 @@ typedef enum HFSearchMode {
HF_SEARCH_MODE_EXHAUSTIVE, // Exhaustive mode: Searches until the best match is found.
} HFSearchMode;
/**
* @brief Primary key mode for face feature management.
*/
typedef enum HFPKMode {
HF_PK_AUTO_INCREMENT = 0, ///< Auto-increment mode for primary key.
HF_PK_MANUAL_INPUT, ///< Manual input mode for primary key.
} HFPKMode;
/**
* @brief Struct for database configuration.
*
* This struct holds the configuration settings for using a database in the face recognition
* context.
* This struct holds the configuration settings for using a database in the face recognition context.
*/
typedef struct HFFeatureHubConfiguration {
HInt32 featureBlockNum; ///< The order of magnitude of face feature database is N * 512, and 20
///< is recommended by default
HInt32 enablePersistence; ///< Flag to enable or disable the use of the database.
HString dbPath; ///< Path to the database file.
float searchThreshold; ///< Threshold for face search
HFSearchMode searchMode; ///< Mode of face search
HFPKMode primaryKeyMode; ///< Primary key mode(The id increment mode is recommended)
HInt32 enablePersistence; ///< Flag to enable or disable the use of the database.
HString persistenceDbPath; ///< Path to the database file.
float searchThreshold; ///< Threshold for face search
HFSearchMode searchMode; ///< Mode of face search
} HFFeatureHubConfiguration;
/**
@@ -400,9 +721,9 @@ HYPER_CAPI_EXPORT extern HResult HFFeatureHubDataDisable();
* This struct associates a custom identifier and a tag with a specific face feature.
*/
typedef struct HFFaceFeatureIdentity {
HInt32 customId; ///< Custom identifier for the face feature.
HString tag; ///< Tag associated with the face feature.
HFaceId id; ///< If you use automatic assignment id mode when inserting, ignore it.
PHFFaceFeature feature; ///< Pointer to the face feature.
// HString tag; ///< Not supported yet
} HFFaceFeatureIdentity, *PHFFaceFeatureIdentity;
/**
@@ -411,7 +732,7 @@ typedef struct HFFaceFeatureIdentity {
typedef struct HFSearchTopKResults {
HInt32 size; ///< The number of faces searched
HPFloat confidence; ///< Search confidence(it has already been filtered once by the threshold)
HPInt32 customIds; ///< fACE customIds
HPFaceId ids; ///< Searched face ids
} HFSearchTopKResults, *PHFSearchTopKResults;
/**
@@ -428,15 +749,68 @@ HYPER_CAPI_EXPORT extern HResult HFFeatureHubFaceSearchThresholdSetting(float th
/**
* @brief Perform a one-to-one comparison of two face features.
* Result is a cosine similarity score, not a percentage similarity.
*
* @param session Handle to the session.
* @param feature1 The first face feature for comparison.
* @param feature2 The second face feature for comparison.
* @param result Pointer to the floating-point value where the comparison result will be stored.
* The result is a cosine similarity score, not a percentage similarity.
* The score ranges from -1 to 1, where 1 indicates identical features,
* 0 indicates orthogonal features, and -1 indicates opposite features.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFaceComparison(HFFaceFeature feature1, HFFaceFeature feature2, HPFloat result);
/**
* @brief Get recommended cosine threshold from loaded resource.
* Use it to determine face similarity. Note: it's just a reference and may not be optimal for your task.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetRecommendedCosineThreshold(HPFloat threshold);
/**
* @brief Convert cosine similarity to percentage similarity.
* This is a nonlinear transformation function. You can adjust curve parameters to map the similarity distribution you need.
* @note The conversion parameters are primarily read from the Resource file configuration, as different models
* have different conversion parameters. The parameters provided in the Resource file are only reference
* values. If they do not meet your specific use case requirements, you can implement your own conversion
* function.
* @param similarity The cosine similarity score.
* @param result Pointer to the floating-point value where the percentage similarity will be stored.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFCosineSimilarityConvertToPercentage(HFloat similarity, HPFloat result);
/**
* @brief Similarity converter configuration.
*/
typedef struct HFSimilarityConverterConfig {
HFloat threshold; ///< If you think that the threshold for judging the same person using cosine is some value such as 0.42,
// you need to convert him to a percentage of 0.6(pass), you can modify it.
HFloat middleScore; ///< Cosine threshold converted to a percentage reference value,
// usually set 0.6 or 0.5, greater than it indicates similar, pass
HFloat steepness; ///< Steepness of the curve, usually set 8.0
HFloat outputMin; ///< Minimum value of output range, usually set 0.01
HFloat outputMax; ///< Maximum value of output range, usually set 1.0
} HFSimilarityConverterConfig, *PHFSimilarityConverterConfig;
/**
* @brief Update the similarity converter configuration.
* @note The default configuration is loaded from the resource file during initialization.
* This function allows you to override those default settings if needed.
* @param config The new similarity converter configuration to apply.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFUpdateCosineSimilarityConverter(HFSimilarityConverterConfig config);
/**
* @brief Get the similarity converter configuration.
* @param config Pointer to the similarity converter configuration to be filled.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFGetCosineSimilarityConverter(PHFSimilarityConverterConfig config);
/**
* @brief Get the length of the face feature.
*
@@ -451,7 +825,7 @@ HYPER_CAPI_EXPORT extern HResult HFGetFeatureLength(HPInt32 num);
* @param featureIdentity The face feature identity to be inserted.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFeatureHubInsertFeature(HFFaceFeatureIdentity featureIdentity);
HYPER_CAPI_EXPORT extern HResult HFFeatureHubInsertFeature(HFFaceFeatureIdentity featureIdentity, HPFaceId allocId);
/**
* @brief Search for the most similar face feature in the features group.
@@ -480,7 +854,7 @@ HYPER_CAPI_EXPORT extern HResult HFFeatureHubFaceSearchTopK(HFFaceFeature search
* @param customId The custom ID of the feature to be removed.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFeatureHubFaceRemove(HInt32 customId);
HYPER_CAPI_EXPORT extern HResult HFFeatureHubFaceRemove(HFaceId id);
/**
* @brief Update a face feature identity in the features group.
@@ -497,7 +871,7 @@ HYPER_CAPI_EXPORT extern HResult HFFeatureHubFaceUpdate(HFFaceFeatureIdentity fe
* @param identity Pointer to the face feature identity to be retrieved.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFeatureHubGetFaceIdentity(HInt32 customId, PHFFaceFeatureIdentity identity);
HYPER_CAPI_EXPORT extern HResult HFFeatureHubGetFaceIdentity(HFaceId customId, PHFFaceFeatureIdentity identity);
/**
* @brief Get the count of face features in the features group.
@@ -514,6 +888,21 @@ HYPER_CAPI_EXPORT extern HResult HFFeatureHubGetFaceCount(HInt32 *count);
*/
HYPER_CAPI_EXPORT extern HResult HFFeatureHubViewDBTable();
/**
* @brief Struct representing the existing ids in the database.
*/
typedef struct HFFeatureHubExistingIds {
HInt32 size; ///< The number of ids
HPFaceId ids; ///< The ids
} HFFeatureHubExistingIds, *PHFFeatureHubExistingIds;
/**
* @brief Get all ids in the database.
* @param ids Output parameter to store the ids.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFFeatureHubGetExistingIds(PHFFeatureHubExistingIds ids);
/************************************************************************
* Face Pipeline
************************************************************************/
@@ -632,32 +1021,32 @@ HYPER_CAPI_EXPORT extern HResult HFFaceQualityDetect(HFSession session, HFFaceBa
/**
* @brief Facial states in the face interaction module.
*/
typedef struct HFFaceIntereactionState {
typedef struct HFFaceInteractionState {
HInt32 num; ///< Number of faces detected.
HPFloat leftEyeStatusConfidence; ///< Left eye state: confidence close to 1 means open, close
///< to 0 means closed.
HPFloat rightEyeStatusConfidence; ///< Right eye state: confidence close to 1 means open, close
///< to 0 means closed.
} HFFaceIntereactionState, *PHFFaceIntereactionState;
} HFFaceInteractionState, *PHFFaceInteractionState;
/**
* @brief Get the prediction results of face interaction.
* @param session Handle to the session.
* @param result Facial state prediction results in the face interaction module.
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionStateResult(HFSession session, PHFFaceIntereactionState result);
HYPER_CAPI_EXPORT extern HResult HFGetFaceInteractionStateResult(HFSession session, PHFFaceInteractionState result);
/**
* @brief Actions detected in the face interaction module.
*/
typedef struct HFFaceIntereactionsActions {
typedef struct HFFaceInteractionsActions {
HInt32 num; ///< Number of actions detected.
HPInt32 normal; ///< Normal actions.
HPInt32 shake; ///< Shake actions.
HPInt32 jawOpen; ///< Jaw open actions.
HPInt32 headRiase; ///< Head raise actions.
HPInt32 headRaise; ///< Head raise actions.
HPInt32 blink; ///< Blink actions.
} HFFaceIntereactionsActions, *PHFFaceIntereactionsActions;
} HFFaceInteractionsActions, *PHFFaceInteractionsActions;
/**
* @brief Get the prediction results of face interaction actions.
@@ -665,7 +1054,7 @@ typedef struct HFFaceIntereactionsActions {
* @param actions Facial action prediction results in the face interaction module.
* @return HResult indicating success or failure of the function call.
*/
HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionActionsResult(HFSession session, PHFFaceIntereactionsActions actions);
HYPER_CAPI_EXPORT extern HResult HFGetFaceInteractionActionsResult(HFSession session, PHFFaceInteractionsActions actions);
/**
* @brief Struct representing face attribute results.
*
@@ -729,6 +1118,21 @@ typedef struct HFInspireFaceVersion {
*/
HYPER_CAPI_EXPORT extern HResult HFQueryInspireFaceVersion(PHFInspireFaceVersion version);
/**
* @brief Struct representing the extended information of the InspireFace library.
*/
typedef struct HFInspireFaceExtendedInformation {
HChar information[256];
// TODO: Add more information
} HFInspireFaceExtendedInformation, *PHFInspireFaceExtendedInformation;
/**
* @brief Get the extended information of the InspireFace library.
*
* This function retrieves the extended information of the InspireFace library.
*/
HYPER_CAPI_EXPORT extern HResult HFQueryInspireFaceExtendedInformation(PHFInspireFaceExtendedInformation information);
/**
* @brief SDK built-in log level mode
* */
@@ -737,10 +1141,8 @@ typedef enum HFLogLevel {
HF_LOG_DEBUG, // Debug level for detailed system information mostly useful for developers
HF_LOG_INFO, // Information level for general system information about operational status
HF_LOG_WARN, // Warning level for non-critical issues that might need attention
HF_LOG_ERROR, // Error level for error events that might still allow the application to
// continue running
HF_LOG_FATAL // Fatal level for severe error events that will presumably lead the application
// to abort
HF_LOG_ERROR, // Error level for error events that might still allow the application to continue running
HF_LOG_FATAL // Fatal level for severe error events that will presumably lead the application to abort
} HFLogLevel;
/**
@@ -753,6 +1155,16 @@ HYPER_CAPI_EXPORT extern HResult HFSetLogLevel(HFLogLevel level);
* */
HYPER_CAPI_EXPORT extern HResult HFLogDisable();
/**
* @brief Print the log.
* @param level The log level.
* @param format The log format.
* @param ... The log arguments.
* @warning The maximum buffer size for log messages is 1024 bytes. Messages longer than this will be truncated.
* @return HResult indicating the success or failure of the operation.
*/
HYPER_CAPI_EXPORT extern HResult HFLogPrint(HFLogLevel level, HFormat format, ...);
/********************************DEBUG Utils****************************************/
/**
@@ -839,4 +1251,4 @@ HYPER_CAPI_EXPORT extern HResult HFDeBugGetUnreleasedStreams(HFImageStream *stre
}
#endif
#endif // HYPERFACEREPO_INSPIREFACE_H
#endif // INSPIREFACE_H

26
cpp-package/inspireface/cpp/inspireface/c_api/inspireface_internal.h
View File

@@ -1,19 +1,23 @@
//
// Created by tunm on 2023/10/3.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef HYPERFACEREPO_INSPIREFACE_INTERNAL_H
#define HYPERFACEREPO_INSPIREFACE_INTERNAL_H
#ifndef INSPIREFACE_INTERNAL_H
#define INSPIREFACE_INTERNAL_H
#include "face_context.h"
#include "face_session.h"
typedef struct HF_FaceAlgorithmSession {
inspire::FaceContext impl; ///< Implementation of the face context.
} HF_FaceAlgorithmSession; ///< Handle for managing face context.
inspire::FaceSession impl; ///< Implementation of the face context.
} HF_FaceAlgorithmSession; ///< Handle for managing face context.
typedef struct HF_CameraStream {
inspire::CameraStream impl; ///< Implementation of the camera stream.
} HF_CameraStream; ///< Handle for managing camera stream.
inspirecv::InspireImageProcess impl; ///< Implementation of the camera stream.
} HF_CameraStream; ///< Handle for managing camera stream.
typedef struct HF_ImageBitmap {
inspirecv::Image impl; ///< Implementation of the image bitmap.
} HF_ImageBitmap; ///< Handle for managing image bitmap.
#endif //HYPERFACEREPO_INSPIREFACE_INTERNAL_H
#endif // INSPIREFACE_INTERNAL_H

37
cpp-package/inspireface/cpp/inspireface/c_api/intypedef.h
View File

@@ -1,28 +1,38 @@
//
// Created by tunm on 2023/10/3.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef HYPERFACEREPO_INTYPEDEF_H
#define HYPERFACEREPO_INTYPEDEF_H
#ifndef INSPIREFACE_INTYPEDEF_H
#define INSPIREFACE_INTYPEDEF_H
#include <stdint.h>
// clang-format off
typedef void* HPVoid; ///< Pointer to Void.
typedef void* HFImageStream; ///< Handle for image.
typedef void* HFSession; ///< Handle for context.
typedef void* HFImageBitmap; ///< Handle for image bitmap.
typedef long HLong; ///< Long integer.
typedef float HFloat; ///< Single-precision floating point.
typedef float* HPFloat; ///< Pointer to Single-precision floating point.
typedef float HFloat; ///< Single-precision floating point.
typedef float* HPFloat; ///< Pointer to Single-precision floating point.
typedef double HDouble; ///< Double-precision floating point.
typedef unsigned char HUInt8; ///< Unsigned 8-bit integer.
typedef unsigned char* HPUInt8; ///< Pointer to unsigned 8-bit integer.
typedef signed int HInt32; ///< Signed 32-bit integer.
typedef signed int HOption; ///< Signed 32-bit integer option.
typedef signed int* HPInt32; ///< Pointer to signed 32-bit integer.
typedef int64_t HFaceId; ///< Face ID type for non-Windows platforms
typedef int64_t* HPFaceId; ///< Pointer to Face ID type for non-Windows platforms
typedef long HResult; ///< Result code.
typedef char* HString; ///< String.
typedef const char* HPath; ///< Const String.
typedef const char* HFormat; ///< Const String.
typedef char HBuffer; ///< Character.
typedef char HChar; ///< Character.
typedef char* HPBuffer; ///< Pointer Character.
typedef long HSize; ///< Size
typedef long* HPSize; ///< Pointer Size
// clang-format on
typedef struct HFaceRect {
HInt32 x; ///< X-coordinate of the top-left corner of the rectangle.
@@ -36,4 +46,15 @@ typedef struct HPoint2f{
HFloat y; ///< Y-coordinate
} HPoint2f;
#endif //HYPERFACEREPO_INTYPEDEF_H
typedef struct HPoint2i{
HInt32 x; ///< X-coordinate
HInt32 y; ///< Y-coordinate
} HPoint2i;
typedef struct HColor {
HFloat r; ///< Red component
HFloat g; ///< Green component
HFloat b; ///< Blue component
} HColor;
#endif //INSPIREFACE_INTYPEDEF_H

25
cpp-package/inspireface/cpp/inspireface/cmake/templates/InspireFaceConfig.cmake.in
View File

@@ -1,25 +0,0 @@
# ===================================================================================
# The InspireFace CMake configuration file
#
# ** File generated automatically, do not modify **
# Usage from an external project:
# In your CMakeLists.txt, add these lines:
#
# find_package(InspireFace REQUIRED)
# include_directories(${InspireFace_INCLUDE_DIRS}) # Not needed for CMake >= 2.8.11
# target_link_libraries(MY_TARGET_NAME ${InspireFace_LIBS})
#
#
#
# This file will define the following variables:
# - InspireFace_LIBS : The list of all imported targets for InspireFace modules.
# - InspireFace_INCLUDE_DIRS : The InspireFace include directories.
#
#
@PACKAGE_INIT@
set(InspireFace_LIBS "")
file(GLOB LIBS "@CMAKE_BINARY_DIR@/InspireFace/lib/*.*")
list(APPEND InspireFace_LIBS ${LIBS})
set(InspireFace_INCLUDE_DIRS "@CMAKE_BINARY_DIR@/InspireFace/include")

54
cpp-package/inspireface/cpp/inspireface/common/face_data/face_data_type.h
View File

@@ -1,15 +1,16 @@
//
// Created by tunm on 2023/9/17.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
// Include guard to prevent double inclusion of this header file
#pragma once
#ifndef HYPERFACEREPO_FACEDATATYPE_H
#define HYPERFACEREPO_FACEDATATYPE_H
#ifndef INSPIRE_FACE_FACEDATATYPE_H
#define INSPIRE_FACE_FACEDATATYPE_H
// Include the necessary header files
#include "../../data_type.h"
#include "../face_info/face_object.h"
#include "../face_info/face_object_internal.h"
// Define the namespace "inspire" for encapsulation
namespace inspire {
@@ -24,8 +25,8 @@ typedef struct Face3DAngle {
} Face3DAngle;
/**
* Struct to represent the rectangle coordinates of a face.
*/
* Struct to represent the rectangle coordinates of a face.
*/
typedef struct FaceRect {
int x; ///< X-coordinate of the top-left corner
int y; ///< Y-coordinate of the top-left corner
@@ -34,16 +35,16 @@ typedef struct FaceRect {
} FaceRect;
/**
* Struct to represent 2D point coordinates.
*/
* Struct to represent 2D point coordinates.
*/
typedef struct Point2F {
float x; ///< X-coordinate
float y; ///< Y-coordinate
} HPoint;
/**
* Struct to represent a 2D transformation matrix.
*/
* Struct to represent a 2D transformation matrix.
*/
typedef struct TransMatrix {
double m00; ///< Element (0,0) of the matrix
double m01; ///< Element (0,1) of the matrix
@@ -54,21 +55,22 @@ typedef struct TransMatrix {
} TransMatrix;
/**
* Struct to represent hyper face data.
*/
* Struct to represent hyper face data.
*/
typedef struct HyperFaceData {
int trackState; ///< Track state
int inGroupIndex; ///< Index within a group
int trackId; ///< Track ID
int trackCount; ///< Track count
FaceRect rect; ///< Face rectangle
TransMatrix trans; ///< Transformation matrix
Point2F keyPoints[5]; ///< Key points (e.g., landmarks)
Face3DAngle face3DAngle; ///< 3D face angles
float quality[5]; ///< Quality values for key points
Point2F densityLandmark[106]; ///< Face density landmark
int trackState; ///< Track state
int inGroupIndex; ///< Index within a group
int trackId; ///< Track ID
int trackCount; ///< Track count
FaceRect rect; ///< Face rectangle
TransMatrix trans; ///< Transformation matrix
Point2F keyPoints[5]; ///< Key points (e.g., landmarks)
Face3DAngle face3DAngle; ///< 3D face angles
float quality[5]; ///< Quality values for key points
Point2F densityLandmark[106]; ///< Face density landmark
int densityLandmarkEnable; ///< Density landmark enable
} HyperFaceData;
} // namespace inspire
} // namespace inspire
#endif //HYPERFACEREPO_FACEDATATYPE_H
#endif // INSPIRE_FACE_FACEDATATYPE_H

128
cpp-package/inspireface/cpp/inspireface/common/face_data/data_tools.h → cpp-package/inspireface/cpp/inspireface/common/face_data/face_serialize_tools.h
View File

@@ -1,14 +1,17 @@
//
// Created by tunm on 2023/9/17.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef INSPIRE_FACE_SERIALIZE_TOOLS_H
#define INSPIRE_FACE_SERIALIZE_TOOLS_H
#ifndef HYPERFACEREPO_DATATOOLS_H
#define HYPERFACEREPO_DATATOOLS_H
#include "opencv2/opencv.hpp"
#include "face_data_type.h"
#include "../face_info/face_object.h"
#include "../face_info/face_object_internal.h"
#include "herror.h"
#include "data_type.h"
#include "track_module/landmark/all.h"
#include <log.h>
// Define the namespace "inspire" for encapsulation
namespace inspire {
@@ -17,35 +20,26 @@ namespace inspire {
* @brief Print the transformation matrix.
* @param matrix The transformation matrix to print.
*/
inline void PrintTransMatrix(const TransMatrix& matrix) {
std::cout << "Transformation Matrix:" << std::endl;
std::cout << "m00: " << matrix.m00 << "\t";
std::cout << "m01: " << matrix.m01 << "\t";
std::cout << "tx: " << matrix.tx << std::endl;
std::cout << "m10: " << matrix.m10 << "\t";
std::cout << "m11: " << matrix.m11 << "\t";
std::cout << "ty: " << matrix.ty << std::endl;
inline void PrintTransformMatrix(const TransMatrix& matrix) {
INSPIRE_LOGI("Transformation Matrix:");
INSPIRE_LOGI("a: %f\tb: %f\ttx: %f", matrix.m00, matrix.m01, matrix.tx);
INSPIRE_LOGI("c: %f\td: %f\tty: %f", matrix.m10, matrix.m11, matrix.ty);
}
/**
* @brief Print HyperFaceData structure.
* @param data The HyperFaceData structure to print.
*/
inline void INSPIRE_API PrintHyperFaceData(const HyperFaceData& data) {
std::cout << "Track State: " << data.trackState << std::endl;
std::cout << "In Group Index: " << data.inGroupIndex << std::endl;
std::cout << "Track ID: " << data.trackId << std::endl;
std::cout << "Track Count: " << data.trackCount << std::endl;
inline void INSPIRE_API PrintHyperFaceDataDetail(const HyperFaceData& data) {
INSPIRE_LOGI("Track State: %d", data.trackState);
INSPIRE_LOGI("In Group Index: %d", data.inGroupIndex);
INSPIRE_LOGI("Track ID: %d", data.trackId);
INSPIRE_LOGI("Track Count: %d", data.trackCount);
std::cout << "Face Rectangle:" << std::endl;
std::cout << "x: " << data.rect.x << "\t";
std::cout << "y: " << data.rect.y << "\t";
std::cout << "width: " << data.rect.width << "\t";
std::cout << "height: " << data.rect.height << std::endl;
PrintTransMatrix(data.trans);
INSPIRE_LOGI("Face Rectangle:");
INSPIRE_LOGI("x: %f\ty: %f\twidth: %f\theight: %f", data.rect.x, data.rect.y, data.rect.width, data.rect.height);
PrintTransformMatrix(data.trans);
}
/**
@@ -54,34 +48,34 @@ inline void INSPIRE_API PrintHyperFaceData(const HyperFaceData& data) {
* @param group_index The group index.
* @return The converted HyperFaceData structure.
*/
inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj, int group_index = -1) {
inline HyperFaceData INSPIRE_API FaceObjectInternalToHyperFaceData(const FaceObjectInternal& obj, int group_index = -1) {
HyperFaceData data;
// Face rect
data.rect.x = obj.bbox_.x;
data.rect.y = obj.bbox_.y;
data.rect.width = obj.bbox_.width;
data.rect.height = obj.bbox_.height;
data.rect.x = obj.bbox_.GetX();
data.rect.y = obj.bbox_.GetY();
data.rect.width = obj.bbox_.GetWidth();
data.rect.height = obj.bbox_.GetHeight();
// Trans matrix
data.trans.m00 = obj.getTransMatrix().at<double>(0, 0);
data.trans.m01 = obj.getTransMatrix().at<double>(0, 1);
data.trans.m10 = obj.getTransMatrix().at<double>(1, 0);
data.trans.m11 = obj.getTransMatrix().at<double>(1, 1);
data.trans.tx = obj.getTransMatrix().at<double>(0, 2);
data.trans.ty = obj.getTransMatrix().at<double>(1, 2);
data.trans.m00 = obj.getTransMatrix().Get(0, 0);
data.trans.m01 = obj.getTransMatrix().Get(0, 1);
data.trans.m10 = obj.getTransMatrix().Get(1, 0);
data.trans.m11 = obj.getTransMatrix().Get(1, 1);
data.trans.tx = obj.getTransMatrix().Get(0, 2);
data.trans.ty = obj.getTransMatrix().Get(1, 2);
// KetPoints five
if (!obj.high_result.lmk.empty()) {
for (int i = 0; i < obj.high_result.lmk.size(); ++i) {
data.keyPoints[i].x = obj.high_result.lmk[i].x;
data.keyPoints[i].y = obj.high_result.lmk[i].y;
data.keyPoints[i].x = obj.high_result.lmk[i].GetX();
data.keyPoints[i].y = obj.high_result.lmk[i].GetY();
}
for (int i = 0; i < 5; ++i) {
data.quality[i] = obj.high_result.lmk_quality[i];
}
// LOGD("HIGHT");
// LOGD("HIGHT");
} else {
for (int i = 0; i < obj.keyPointFive.size(); ++i) {
data.keyPoints[i].x = obj.keyPointFive[i].x;
data.keyPoints[i].y = obj.keyPointFive[i].y;
data.keyPoints[i].x = obj.keyPointFive[i].GetX();
data.keyPoints[i].y = obj.keyPointFive[i].GetY();
}
for (int i = 0; i < 5; ++i) {
data.quality[i] = -1.0f;
@@ -96,15 +90,17 @@ inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj
data.face3DAngle.pitch = obj.high_result.pitch;
data.face3DAngle.roll = obj.high_result.roll;
data.face3DAngle.yaw = obj.high_result.yaw;
const auto &lmk = obj.landmark_smooth_aux_.back();
for (size_t i = 0; i < lmk.size(); i++)
{
data.densityLandmark[i].x = lmk[i].x;
data.densityLandmark[i].y = lmk[i].y;
// Density Landmark
if (!obj.landmark_smooth_aux_.empty()) {
data.densityLandmarkEnable = 1;
const auto& lmk = obj.landmark_smooth_aux_.back();
for (size_t i = 0; i < FaceLandmarkAdapt::NUM_OF_LANDMARK; i++) {
data.densityLandmark[i].x = lmk[i].GetX();
data.densityLandmark[i].y = lmk[i].GetY();
}
} else {
data.densityLandmarkEnable = 0;
}
return data;
}
@@ -114,14 +110,8 @@ inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj
* @param trans The TransMatrix to convert.
* @return The converted cv::Mat.
*/
inline cv::Mat INSPIRE_API TransMatrixToMat(const TransMatrix& trans) {
cv::Mat mat(2, 3, CV_64F);
mat.at<double>(0, 0) = trans.m00;
mat.at<double>(0, 1) = trans.m01;
mat.at<double>(1, 0) = trans.m10;
mat.at<double>(1, 1) = trans.m11;
mat.at<double>(0, 2) = trans.tx;
mat.at<double>(1, 2) = trans.ty;
inline inspirecv::TransformMatrix INSPIRE_API TransformMatrixToInternalMatrix(const TransMatrix& trans) {
inspirecv::TransformMatrix mat = inspirecv::TransformMatrix::Create(trans.m00, trans.m01, trans.tx, trans.m10, trans.m11, trans.ty);
return mat;
}
@@ -130,8 +120,8 @@ inline cv::Mat INSPIRE_API TransMatrixToMat(const TransMatrix& trans) {
* @param faceRect The FaceRect to convert.
* @return The converted cv::Rect.
*/
inline cv::Rect INSPIRE_API FaceRectToRect(const FaceRect& faceRect) {
return {faceRect.x, faceRect.y, faceRect.width, faceRect.height};
inline inspirecv::Rect2i INSPIRE_API FaceRectToInternalRect(const FaceRect& faceRect) {
return inspirecv::Rect2i(faceRect.x, faceRect.y, faceRect.width, faceRect.height);
}
/**
@@ -139,8 +129,8 @@ inline cv::Rect INSPIRE_API FaceRectToRect(const FaceRect& faceRect) {
* @param point The Point2F to convert.
* @return The converted cv::Point2f.
*/
inline cv::Point2f INSPIRE_API HPointToPoint2f(const Point2F& point) {
return {point.x, point.y};
inline inspirecv::Point2f INSPIRE_API HPointToInternalPoint2f(const Point2F& point) {
return inspirecv::Point2f(point.x, point.y);
}
/**
@@ -149,7 +139,7 @@ inline cv::Point2f INSPIRE_API HPointToPoint2f(const Point2F& point) {
* @param byteArray The output byte stream.
* @return The result code.
*/
inline int32_t INSPIRE_API SerializeHyperFaceData(const HyperFaceData& data, ByteArray& byteArray) {
inline int32_t INSPIRE_API RunSerializeHyperFaceData(const HyperFaceData& data, ByteArray& byteArray) {
byteArray.reserve(sizeof(data));
// Serialize the HyperFaceData structure itself
@@ -165,7 +155,7 @@ inline int32_t INSPIRE_API SerializeHyperFaceData(const HyperFaceData& data, Byt
* @param data The output HyperFaceData structure.
* @return The result code.
*/
inline int32_t INSPIRE_API DeserializeHyperFaceData(const ByteArray& byteArray, HyperFaceData &data) {
inline int32_t INSPIRE_API RunDeserializeHyperFaceData(const ByteArray& byteArray, HyperFaceData& data) {
// Check if the byte stream size is sufficient
if (byteArray.size() >= sizeof(data)) {
// Copy data from the byte stream to the HyperFaceData structure
@@ -185,7 +175,7 @@ inline int32_t INSPIRE_API DeserializeHyperFaceData(const ByteArray& byteArray,
* @param data The output HyperFaceData structure.
* @return The result code.
*/
inline int32_t INSPIRE_API DeserializeHyperFaceData(const char* byteArray, size_t byteCount, HyperFaceData& data) {
inline int32_t INSPIRE_API RunDeserializeHyperFaceData(const char* byteArray, size_t byteCount, HyperFaceData& data) {
// Check if the byte stream size is sufficient
if (byteCount >= sizeof(data)) {
// Copy data from the byte stream to the HyperFaceData structure
@@ -198,5 +188,5 @@ inline int32_t INSPIRE_API DeserializeHyperFaceData(const char* byteArray, size_
return HSUCCEED;
}
} // namespace hyper
#endif //HYPERFACEREPO_DATATOOLS_H
} // namespace inspire
#endif // INSPIRE_FACE_SERIALIZE_TOOLS_H

17
cpp-package/inspireface/cpp/inspireface/common/face_info/all.h
View File

@@ -1,10 +1,13 @@
//
// Created by tunm on 2023/8/29.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef HYPERFACEREPO_FO_ALL_H
#define HYPERFACEREPO_FO_ALL_H
#ifndef INSPIRE_FACE_FO_ALL_H
#define INSPIRE_FACE_FO_ALL_H
#include "face_object.h"
#include "face_object_internal.h"
#include "face_action_data.h"
#include "face_process.h"
#endif //HYPERFACEREPO_FO_ALL_H
#endif // INSPIRE_FACE_FO_ALL_H

75
cpp-package/inspireface/cpp/inspireface/common/face_info/face_action.h → cpp-package/inspireface/cpp/inspireface/common/face_info/face_action_data.h
View File

@@ -1,33 +1,27 @@
#ifndef INSPIRSE_FACE_FACE_ACTION_H
#define INSPIRSE_FACE_FACE_ACTION_H
#ifndef INSPIRSE_FACE_FACE_ACTION_DATA_H
#define INSPIRSE_FACE_FACE_ACTION_DATA_H
#include <iostream>
#include "opencv2/opencv.hpp"
#include <inspirecv/inspirecv.h>
#include "middleware/utils.h"
#include "data_type.h"
#include "track_module/landmark/face_landmark.h"
#include "track_module/landmark/face_landmark_adapt.h"
namespace inspire {
enum FACE_ACTION {
NORMAL = 0,
SHAKE = 0,
BLINK = 1,
JAW_OPEN = 2,
RAISE_HEAD = 3
};
enum FACE_ACTIONS { ACT_NORMAL = 0, ACT_SHAKE = 0, ACT_BLINK = 1, ACT_JAW_OPEN = 2, ACT_RAISE_HEAD = 3 };
typedef struct FaceActions{
typedef struct FaceActionList {
int normal = 0;
int shake = 0;
int blink = 0;
int jawOpen = 0;
int raiseHead = 0;
} FaceActions;
} FaceActionList;
class INSPIRE_API FaceActionAnalyse {
class INSPIRE_API FaceActionPredictor {
public:
FaceActionAnalyse(int record_list_length) {
FaceActionPredictor(int record_list_length) {
record_list.resize(record_list_length);
record_list_euler.resize(record_list_length);
record_list_eyes.resize(record_list_length);
@@ -35,9 +29,8 @@ public:
index = 0;
}
void RecordActionFrame(const std::vector<cv::Point2f> &landmark,
const cv::Vec3f &euler_angle,
const cv::Vec2f &eyes_status) {
void RecordActionFrame(const std::vector<inspirecv::Point2f> &landmark, const inspirecv::Vec3f &euler_angle,
const inspirecv::Vec2f &eyes_status) {
MoveRecordList();
record_list[0] = landmark;
record_list_euler[0] = euler_angle;
@@ -53,14 +46,14 @@ public:
record_list_eyes.clear();
record_list_eyes.resize(record_size);
index = 0;
}
}
FaceActions AnalysisFaceAction() {
FaceActions actionRecord;
FaceActionList AnalysisFaceAction() {
FaceActionList actionRecord;
actions.clear();
eye_state_list.clear();
if (index < record_list.size()) {
actions.push_back(NORMAL);
actions.push_back(ACT_NORMAL);
actionRecord.normal = 1;
} else {
for (int i = 0; i < record_list_eyes.size(); i++) {
@@ -70,15 +63,12 @@ public:
}
// count mouth aspect ratio
float mouth_widthwise_d =
PointDistance(record_list[0][FaceLandmark::MOUTH_LEFT_CORNER],
record_list[0][FaceLandmark::MOUTH_RIGHT_CORNER]);
float mouth_heightwise_d =
PointDistance(record_list[0][FaceLandmark::MOUTH_UPPER],
record_list[0][FaceLandmark::MOUTH_LOWER]);
float mouth_widthwise_d = record_list[0][FaceLandmarkAdapt::MOUTH_LEFT_CORNER].Distance(record_list[0][FaceLandmarkAdapt::MOUTH_RIGHT_CORNER]);
float mouth_heightwise_d = record_list[0][FaceLandmarkAdapt::MOUTH_UPPER].Distance(record_list[0][FaceLandmarkAdapt::MOUTH_LOWER]);
float mouth_aspect_ratio = mouth_heightwise_d / mouth_widthwise_d;
if (mouth_aspect_ratio > 0.3) {
actions.push_back(JAW_OPEN);
actions.push_back(ACT_JAW_OPEN);
actionRecord.jawOpen = 1;
}
@@ -92,9 +82,8 @@ public:
counter_eye_open += 1;
}
}
if (counter_eye_close > 0 && counter_eye_open > 2 &&
record_list_euler[0][1] > -6 && record_list_euler[0][0] < 6) {
actions.push_back(BLINK);
if (counter_eye_close > 0 && counter_eye_open > 2 && record_list_euler[0][1] > -6 && record_list_euler[0][0] < 6) {
actions.push_back(ACT_BLINK);
actionRecord.blink = 1;
Reset();
}
@@ -110,20 +99,19 @@ public:
}
}
if (counter_head_shake_left && counter_head_shake_right) {
actions.push_back(SHAKE);
actions.push_back(ACT_SHAKE);
actionRecord.shake = 1;
}
if (record_list_euler[0][0] > 10) {
actions.push_back(RAISE_HEAD);
actions.push_back(ACT_RAISE_HEAD);
actionRecord.raiseHead = 1;
}
}
return actionRecord;
}
std::vector<FACE_ACTION> GetActions() const {
std::vector<FACE_ACTIONS> GetActions() const {
return actions;
}
@@ -140,17 +128,16 @@ private:
}
}
std::vector<std::vector<cv::Point2f>> record_list;
std::vector<cv::Vec3f> record_list_euler;
std::vector<cv::Vec2f> record_list_eyes;
std::vector<std::pair<float, float>> eye_state_list; // pair left right
std::vector<std::vector<inspirecv::Point2f>> record_list;
std::vector<inspirecv::Vec3f> record_list_euler;
std::vector<inspirecv::Vec2f> record_list_eyes;
std::vector<std::pair<float, float>> eye_state_list; // pair left right
std::vector<float> mouth_state_list;
std::vector<FACE_ACTION> actions;
std::vector<FACE_ACTIONS> actions;
int record_size;
int index;
};
} // namespace inspire
} // namespace inspire
#endif
#endif // INSPIRSE_FACE_FACE_ACTION_DATA_H

332
cpp-package/inspireface/cpp/inspireface/common/face_info/face_object.h
View File

@@ -1,332 +0,0 @@
#ifndef FACE_INFO_H
#define FACE_INFO_H
#include <memory>
#include <utility>
//#include "face_action.h"
#include "opencv2/opencv.hpp"
#include "middleware/utils.h"
#include "data_type.h"
#include "face_process.h"
#include "track_module/quality/face_pose_quality.h"
#include "face_action.h"
namespace inspire {
enum TRACK_STATE {
UNTRACKING = -1, DETECT = 0, READY = 1, TRACKING = 2
};
class INSPIRE_API FaceObject {
public:
FaceObject(int instance_id, cv::Rect bbox, int num_landmark = 106) {
face_id_ = instance_id;
landmark_.resize(num_landmark);
bbox_ = std::move(bbox);
tracking_state_ = DETECT;
confidence_ = 1.0;
tracking_count_ = 0;
pose_euler_angle_.resize(3);
keyPointFive.resize(5);
face_action_ = std::make_shared<FaceActionAnalyse>(10);
}
void UpdateMatrix(const cv::Mat &matrix) {
assert(trans_matrix_.rows == 2 && trans_matrix_.cols == 3);
double a00 = matrix.at<double>(0, 0);
double a01 = matrix.at<double>(0, 1);
double a10 = matrix.at<double>(1, 0);
double a11 = matrix.at<double>(1, 1);
double t1x = matrix.at<double>(0, 2);
double t1y = matrix.at<double>(1, 2);
double m00 = trans_matrix_.at<double>(0, 0);
double m01 = trans_matrix_.at<double>(0, 1);
double m10 = trans_matrix_.at<double>(1, 0);
double m11 = trans_matrix_.at<double>(1, 1);
double t0x = trans_matrix_.at<double>(0, 2);
double t0y = trans_matrix_.at<double>(1, 2);
double n_m00 = a00 * m00 + a01 * m10;
double n_m01 = a00 * m01 + a01 * m11;
double n_m02 = a00 * t0x + a01 * t0y + t1x;
double n_m10 = a10 * m00 + a11 * m10;
double n_m11 = a10 * m01 + a11 * m11;
double n_m12 = a10 * t0x + a11 * t0y + t1y;
trans_matrix_.at<double>(0, 0) = n_m00;
trans_matrix_.at<double>(0, 1) = n_m01;
trans_matrix_.at<double>(0, 2) = n_m02;
trans_matrix_.at<double>(1, 0) = n_m10;
trans_matrix_.at<double>(1, 1) = n_m11;
trans_matrix_.at<double>(1, 2) = n_m12;
}
void SetLandmark(const std::vector<cv::Point2f> &lmk, bool update_rect = true,
bool update_matrix = true) {
if (lmk.size() != landmark_.size()) {
INSPIRE_LOGW("The SetLandmark function displays an exception indicating that the lmk number does not match");
return;
}
std::copy(lmk.begin(), lmk.end(), landmark_.begin());
DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, 106 * 2, 0.06);
// cv::Vec3d euler_angle;
EstimateHeadPose(landmark_, euler_angle_);
// DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, 106 * 2, 0.06);
if (update_rect)
bbox_ = cv::boundingRect(lmk);
if (update_matrix && tracking_state_ == TRACKING) {
// pass
}
keyPointFive[0] = landmark_[55];
keyPointFive[1] = landmark_[105];
keyPointFive[2] = landmark_[69];
keyPointFive[3] = landmark_[45];
keyPointFive[4] = landmark_[50];
}
void setAlignMeanSquareError(const std::vector<cv::Point2f> &lmk_5) {
float src_pts[] = {30.2946, 51.6963, 65.5318, 51.5014, 48.0252,
71.7366, 33.5493, 92.3655, 62.7299, 92.2041};
for (int i = 0; i < 5; i++) {
*(src_pts + 2 * i) += 8.0;
}
float sum = 0;
for (int i = 0; i < lmk_5.size(); i++) {
float l2 = L2norm(src_pts[i * 2 + 0], src_pts[i * 2 + 1], lmk_5[i].x, lmk_5[i].y);
sum += l2;
}
align_mse_ = sum / 5.0f;
}
// 增加跟踪次数
void IncrementTrackingCount() {
tracking_count_++;
}
// 获取跟踪次数
int GetTrackingCount() const {
return tracking_count_;
}
float GetAlignMSE() const { return align_mse_; }
std::vector<cv::Point2f> GetLanmdark() const { return landmark_; }
cv::Rect GetRect() const { return bbox_; }
cv::Rect GetRectSquare(float padding_ratio = 0.0) const {
int cx = bbox_.x + bbox_.width / 2;
int cy = bbox_.y + bbox_.height / 2;
int R = std::max(bbox_.width, bbox_.height) / 2;
int R_padding = static_cast<int>(R * (1 + padding_ratio));
int x1 = cx - R_padding;
int y1 = cy - R_padding;
int x2 = cx + R_padding;
int y2 = cy + R_padding;
int width = x2 - x1;
int height = y2 - y1;
assert(width > 0);
assert(height > 0);
assert(height == width);
cv::Rect box_square(x1, y1, width, height);
return box_square;
}
FaceActions UpdateFaceAction() {
cv::Vec3f euler(high_result.pitch, high_result.yaw, high_result.roll);
cv::Vec2f eyes(left_eye_status_.back(), right_eye_status_.back());
face_action_->RecordActionFrame(landmark_, euler, eyes);
return face_action_->AnalysisFaceAction();
}
void DisableTracking() { tracking_state_ = UNTRACKING; }
void EnableTracking() { tracking_state_ = TRACKING; }
void ReadyTracking() { tracking_state_ = READY; }
TRACK_STATE TrackingState() const { return tracking_state_; }
float GetConfidence() const { return confidence_; }
void SetConfidence(float confidence) { confidence_ = confidence; }
int GetTrackingId() const { return face_id_; }
const cv::Mat &getTransMatrix() const { return trans_matrix_; }
void setTransMatrix(const cv::Mat &transMatrix) {
transMatrix.copyTo(trans_matrix_);
}
static float L2norm(float x0, float y0, float x1, float y1) {
return sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
}
void RequestFaceAction(
std::vector<cv::Point2f> &landmarks,
std::vector<std::vector<cv::Point2f>> &landmarks_lastNframes,
int lm_length, float h) {
int n = 5;
std::vector<cv::Point2f> landmarks_temp;
landmarks_temp.assign(landmarks.begin(), landmarks.end());
if (landmarks_lastNframes.size() == n) {
for (int i = 0; i < lm_length / 2; i++) {
float sum_d = 1;
float max_d = 0;
for (int j = 0; j < n; j++) {
float d = L2norm(landmarks_temp[i].x, landmarks_temp[i].y,
landmarks_lastNframes[j][i].x,
landmarks_lastNframes[j][i].y);
if (d > max_d)
max_d = d;
}
for (int j = 0; j < n; j++) {
float d = exp(-max_d * (n - j) * h);
sum_d += d;
landmarks[i].x = landmarks[i].x + d * landmarks_lastNframes[j][i].x;
landmarks[i].y = landmarks[i].y + d * landmarks_lastNframes[j][i].y;
}
landmarks[i].x = landmarks[i].x / sum_d;
landmarks[i].y = landmarks[i].y / sum_d;
}
}
std::vector<cv::Point2f> landmarks_frame;
for (int i = 0; i < lm_length / 2; i++) {
landmarks_frame.push_back(cv::Point2f(landmarks[i].x, landmarks[i].y));
}
landmarks_lastNframes.push_back(landmarks_frame);
if (landmarks_lastNframes.size() > 5)
landmarks_lastNframes.erase(landmarks_lastNframes.begin());
}
void DynamicSmoothParamUpdate(
std::vector<cv::Point2f> &landmarks,
std::vector<std::vector<cv::Point2f>> &landmarks_lastNframes,
int lm_length, float h) {
int n = 5;
std::vector<cv::Point2f> landmarks_temp;
landmarks_temp.assign(landmarks.begin(), landmarks.end());
if (landmarks_lastNframes.size() == n) {
for (int i = 0; i < lm_length / 2; i++) {
float sum_d = 1;
float max_d = 0;
for (int j = 0; j < n; j++) {
float d = L2norm(landmarks_temp[i].x, landmarks_temp[i].y,
landmarks_lastNframes[j][i].x,
landmarks_lastNframes[j][i].y);
if (d > max_d)
max_d = d;
}
for (int j = 0; j < n; j++) {
float d = exp(-max_d * (n - j) * h);
sum_d += d;
landmarks[i].x = landmarks[i].x + d * landmarks_lastNframes[j][i].x;
landmarks[i].y = landmarks[i].y + d * landmarks_lastNframes[j][i].y;
}
landmarks[i].x = landmarks[i].x / sum_d;
landmarks[i].y = landmarks[i].y / sum_d;
}
}
std::vector<cv::Point2f> landmarks_frame;
for (int i = 0; i < lm_length / 2; i++) {
landmarks_frame.push_back(cv::Point2f(landmarks[i].x, landmarks[i].y));
}
landmarks_lastNframes.push_back(landmarks_frame);
if (landmarks_lastNframes.size() > 5)
landmarks_lastNframes.erase(landmarks_lastNframes.begin());
}
public:
std::vector<cv::Point2f> landmark_;
std::vector<std::vector<cv::Point2f>> landmark_smooth_aux_;
cv::Rect bbox_;
cv::Vec3f euler_angle_;
std::vector<float> pose_euler_angle_;
float align_mse_{};
const cv::Vec3f &getEulerAngle() const { return euler_angle_; }
const std::vector<float> &getPoseEulerAngle() const { return pose_euler_angle_; }
void setPoseEulerAngle(const std::vector<float> &poseEulerAngle) {
pose_euler_angle_[0] = poseEulerAngle[0];
pose_euler_angle_[1] = poseEulerAngle[1];
pose_euler_angle_[2] = poseEulerAngle[2];
if (abs(pose_euler_angle_[0]) < 0.5 && abs(pose_euler_angle_[1]) < 0.48) {
is_standard_ = true;
}
}
bool isStandard() const {
return is_standard_;
}
const cv::Rect &getBbox() const { return bbox_; }
std::vector<cv::Point2f> getRotateLandmark(int height, int width, int rotate = 0) {
if (rotate != 0) {
std::vector<cv::Point2f> result = RotatePoints(landmark_, rotate, cv::Size(height, width));
return result;
} else {
return GetLanmdark();
}
}
cv::Rect getRotateBbox(int height, int width, int rotate = 0, bool use_flip = false) {
if (rotate != 0) {
cv::Rect src_bbox = bbox_;
std::vector<cv::Point2f> points;
cv::Rect trans_rect;
RotateRect(src_bbox, points, trans_rect, rotate, cv::Size(height, width));
if (use_flip)
trans_rect = flipRectWidth(trans_rect, cv::Size(width, height));
return trans_rect;
} else {
return getBbox();
}
}
void setBbox(const cv::Rect &bbox) { bbox_ = bbox; }
cv::Mat trans_matrix_;
float confidence_;
cv::Rect detect_bbox_;
int tracking_count_; // 跟踪次数
bool is_standard_;
FacePoseQualityResult high_result;
FaceProcess faceProcess;
std::vector<Point2f> keyPointFive;
void setId(int id) {
face_id_ = id;
}
std::vector<float> left_eye_status_;
std::vector<float> right_eye_status_;
private:
TRACK_STATE tracking_state_;
std::shared_ptr<FaceActionAnalyse> face_action_;
int face_id_;
};
typedef std::vector<FaceObject> FaceObjectList;
} // namespace hyper
#endif // FACE_INFO_H

273
cpp-package/inspireface/cpp/inspireface/common/face_info/face_object_internal.h Executable file
View File

@@ -0,0 +1,273 @@
#ifndef INSPIRE_FACE_FACE_INFO_INTERNAL_H
#define INSPIRE_FACE_FACE_INFO_INTERNAL_H
#include <memory>
#include <utility>
#include <inspirecv/inspirecv.h>
#include "middleware/utils.h"
#include "data_type.h"
#include "face_process.h"
#include "face_action_data.h"
#include "track_module/quality/face_pose_quality_adapt.h"
namespace inspire {
enum ISF_TRACK_STATE { ISF_UNTRACKING = -1, ISF_DETECT = 0, ISF_READY = 1, ISF_TRACKING = 2 };
class INSPIRE_API FaceObjectInternal {
public:
FaceObjectInternal(int instance_id, inspirecv::Rect2i bbox, int num_landmark = 106) {
face_id_ = instance_id;
landmark_.resize(num_landmark);
bbox_ = std::move(bbox);
tracking_state_ = ISF_DETECT;
confidence_ = 1.0;
tracking_count_ = 0;
pose_euler_angle_.resize(3);
keyPointFive.resize(5);
face_action_ = std::make_shared<FaceActionPredictor>(10);
num_of_dense_landmark_ = num_landmark;
}
void SetLandmark(const std::vector<inspirecv::Point2f> &lmk, bool update_rect = true, bool update_matrix = true, float h = 0.06f, int n = 5,
int num_of_lmk = 106 * 2) {
// if (lmk.size() != landmark_.size()) {
// INSPIRE_LOGW("The SetLandmark function displays an exception indicating that the lmk number does not match");
// return;
// }
std::copy(lmk.begin(), lmk.end(), landmark_.begin());
DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, num_of_lmk, h, n);
// std::cout << "smooth ratio: " << h << " num smooth cache frame: " << n << std::endl;
// cv::Vec3d euler_angle;
// EstimateHeadPose(landmark_, euler_angle_);
// DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, 106 * 2, 0.06);
if (update_rect)
bbox_ = inspirecv::MinBoundingRect(lmk).As<int>();
if (update_matrix && tracking_state_ == ISF_TRACKING) {
// pass
}
keyPointFive[0] = landmark_[55];
keyPointFive[1] = landmark_[105];
keyPointFive[2] = landmark_[69];
keyPointFive[3] = landmark_[45];
keyPointFive[4] = landmark_[50];
}
void setAlignMeanSquareError(const std::vector<inspirecv::Point2f> &lmk_5) {
float src_pts[] = {30.2946, 51.6963, 65.5318, 51.5014, 48.0252, 71.7366, 33.5493, 92.3655, 62.7299, 92.2041};
for (int i = 0; i < 5; i++) {
*(src_pts + 2 * i) += 8.0;
}
float sum = 0;
for (int i = 0; i < lmk_5.size(); i++) {
float l2 = L2norm(src_pts[i * 2 + 0], src_pts[i * 2 + 1], lmk_5[i].GetX(), lmk_5[i].GetY());
sum += l2;
}
align_mse_ = sum / 5.0f;
}
// Increment tracking count
void IncrementTrackingCount() {
tracking_count_++;
}
// Get tracking count
int GetTrackingCount() const {
return tracking_count_;
}
float GetAlignMSE() const {
return align_mse_;
}
std::vector<inspirecv::Point2f> GetLanmdark() const {
return landmark_;
}
inspirecv::Rect2i GetRect() const {
return bbox_;
}
inspirecv::Rect2i GetRectSquare(float padding_ratio = 0.0) const {
int cx = bbox_.GetX() + bbox_.GetWidth() / 2;
int cy = bbox_.GetY() + bbox_.GetHeight() / 2;
int R = std::max(bbox_.GetWidth(), bbox_.GetHeight()) / 2;
int R_padding = static_cast<int>(R * (1 + padding_ratio));
int x1 = cx - R_padding;
int y1 = cy - R_padding;
int x2 = cx + R_padding;
int y2 = cy + R_padding;
int width = x2 - x1;
int height = y2 - y1;
assert(width > 0);
assert(height > 0);
assert(height == width);
inspirecv::Rect2i box_square(x1, y1, width, height);
return box_square;
}
FaceActionList UpdateFaceAction() {
inspirecv::Vec3f euler{high_result.pitch, high_result.yaw, high_result.roll};
inspirecv::Vec2f eyes{left_eye_status_.back(), right_eye_status_.back()};
face_action_->RecordActionFrame(landmark_, euler, eyes);
return face_action_->AnalysisFaceAction();
}
void DisableTracking() {
tracking_state_ = ISF_UNTRACKING;
}
void EnableTracking() {
tracking_state_ = ISF_TRACKING;
}
void ReadyTracking() {
tracking_state_ = ISF_READY;
}
ISF_TRACK_STATE TrackingState() const {
return tracking_state_;
}
float GetConfidence() const {
return confidence_;
}
void SetConfidence(float confidence) {
confidence_ = confidence;
}
int GetTrackingId() const {
return face_id_;
}
const inspirecv::TransformMatrix &getTransMatrix() const {
return trans_matrix_;
}
const inspirecv::TransformMatrix &getTransMatrixExtensive() const {
return trans_matrix_extensive_;
}
void setTransMatrix(const inspirecv::TransformMatrix &transMatrix) {
trans_matrix_ = transMatrix.Clone();
}
void setTransMatrixExtensive(const inspirecv::TransformMatrix &transMatrixExtensive) {
trans_matrix_extensive_ = transMatrixExtensive.Clone();
}
static float L2norm(float x0, float y0, float x1, float y1) {
return sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
}
void DynamicSmoothParamUpdate(std::vector<inspirecv::Point2f> &landmarks, std::vector<std::vector<inspirecv::Point2f>> &landmarks_lastNframes,
int lm_length, float h = 0.06f, int n = 5) {
std::vector<inspirecv::Point2f> landmarks_temp;
landmarks_temp.assign(landmarks.begin(), landmarks.end());
if (landmarks_lastNframes.size() == n) {
for (int i = 0; i < lm_length / 2; i++) {
float sum_d = 1;
float max_d = 0;
for (int j = 0; j < n; j++) {
float d = L2norm(landmarks_temp[i].GetX(), landmarks_temp[i].GetY(), landmarks_lastNframes[j][i].GetX(),
landmarks_lastNframes[j][i].GetY());
if (d > max_d)
max_d = d;
}
for (int j = 0; j < n; j++) {
float d = exp(-max_d * (n - j) * h);
sum_d += d;
landmarks[i].SetX(landmarks[i].GetX() + d * landmarks_lastNframes[j][i].GetX());
landmarks[i].SetY(landmarks[i].GetY() + d * landmarks_lastNframes[j][i].GetY());
}
landmarks[i].SetX(landmarks[i].GetX() / sum_d);
landmarks[i].SetY(landmarks[i].GetY() / sum_d);
}
}
std::vector<inspirecv::Point2f> landmarks_frame;
for (int i = 0; i < lm_length / 2; i++) {
landmarks_frame.push_back(inspirecv::Point2f(landmarks[i].GetX(), landmarks[i].GetY()));
}
landmarks_lastNframes.push_back(landmarks_frame);
if (landmarks_lastNframes.size() > n)
landmarks_lastNframes.erase(landmarks_lastNframes.begin());
}
public:
std::vector<inspirecv::Point2f> landmark_;
std::vector<std::vector<inspirecv::Point2f>> landmark_smooth_aux_;
inspirecv::Rect2i bbox_;
inspirecv::Vec3f euler_angle_;
std::vector<float> pose_euler_angle_;
int num_of_dense_landmark_;
float align_mse_{};
const inspirecv::Vec3f &getEulerAngle() const {
return euler_angle_;
}
const std::vector<float> &getPoseEulerAngle() const {
return pose_euler_angle_;
}
void setPoseEulerAngle(const std::vector<float> &poseEulerAngle) {
pose_euler_angle_[0] = poseEulerAngle[0];
pose_euler_angle_[1] = poseEulerAngle[1];
pose_euler_angle_[2] = poseEulerAngle[2];
if (abs(pose_euler_angle_[0]) < 0.5 && abs(pose_euler_angle_[1]) < 0.48) {
is_standard_ = true;
}
}
bool isStandard() const {
return is_standard_;
}
const inspirecv::Rect2i &getBbox() const {
return bbox_;
}
void setBbox(const inspirecv::Rect2i &bbox) {
bbox_ = bbox;
}
inspirecv::TransformMatrix trans_matrix_;
inspirecv::TransformMatrix trans_matrix_extensive_;
float confidence_;
inspirecv::Rect2i detect_bbox_;
int tracking_count_; // Tracking count
bool is_standard_;
FacePoseQualityAdaptResult high_result;
FaceProcess faceProcess;
std::vector<inspirecv::Point2f> keyPointFive;
void setId(int id) {
face_id_ = id;
}
std::vector<float> left_eye_status_;
std::vector<float> right_eye_status_;
private:
ISF_TRACK_STATE tracking_state_;
std::shared_ptr<FaceActionPredictor> face_action_;
int face_id_;
};
typedef std::vector<FaceObjectInternal> FaceObjectInternalList;
} // namespace inspire
#endif // INSPIRE_FACE_FACE_INFO_INTERNAL_H

24
cpp-package/inspireface/cpp/inspireface/common/face_info/face_process.h
View File

@@ -1,11 +1,12 @@
//
// Created by tunm on 2023/9/12.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
// Include guard to prevent double inclusion of this header file
#pragma once
#ifndef HYPERFACEREPO_FACEPROCESS_H
#define HYPERFACEREPO_FACEPROCESS_H
#ifndef INSPIRE_FACE_FACEPROCESS_H
#define INSPIRE_FACE_FACEPROCESS_H
// Include the necessary header file "data_type.h"
#include "data_type.h"
@@ -23,8 +24,8 @@ typedef enum MaskInfo {
} MaskInfo;
/**
* Enumeration to represent different RGB liveness information.
*/
* Enumeration to represent different RGB liveness information.
*/
typedef enum RGBLivenessInfo {
UNKNOWN_RGB_LIVENESS = -1, ///< Unknown RGB liveness status
LIVENESS_FAKE = 0, ///< Fake liveness
@@ -32,8 +33,8 @@ typedef enum RGBLivenessInfo {
} RGBLivenessInfo;
/**
* Class definition for FaceProcess.
*/
* Class definition for FaceProcess.
*/
class INSPIRE_API FaceProcess {
public:
/**
@@ -45,9 +46,8 @@ public:
* Member variable to store RGB liveness information, initialized to UNKNOWN_RGB_LIVENESS.
*/
RGBLivenessInfo rgbLivenessInfo = UNKNOWN_RGB_LIVENESS;
};
} // namespace hyper
} // namespace inspire
#endif //HYPERFACEREPO_FACEPROCESS_H
#endif // INSPIRE_FACE_FACEPROCESS_H

153
cpp-package/inspireface/cpp/inspireface/data_type.h
View File

@@ -1,9 +1,10 @@
//
// Created by tunm on 2023/5/5.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef HYPERFACE_DATATYPE_H
#define HYPERFACE_DATATYPE_H
#ifndef INSPIRE_FACE_DATATYPE_H
#define INSPIRE_FACE_DATATYPE_H
#include <cstdint>
#if defined(_WIN32) && (defined(_DEBUG) || defined(DEBUG))
@@ -15,7 +16,7 @@
#define INSPIRE_API
#endif
#include <opencv2/opencv.hpp>
#include <inspirecv/inspirecv.h>
#ifndef M_PI
#define M_PI 3.14159265358979323846264338327950288
@@ -24,138 +25,138 @@
namespace inspire {
/**
* @defgroup DataType Definitions
* @brief Defines various data types used in the HyperFace project.
* @{
*/
* @defgroup DataType Definitions
* @brief Defines various data types used in the HyperFace project.
* @{
*/
#if !defined(int64)
/** @typedef int64
* @brief 64-bit integer type.
*/
* @brief 64-bit integer type.
*/
typedef int64_t int64;
#endif
#if !defined(uint64)
/** @typedef uint64
* @brief 64-bit unsigned integer type.
*/
* @brief 64-bit unsigned integer type.
*/
typedef uint64_t uint64;
#endif
#if !defined(int32)
/** @typedef int32
* @brief 32-bit integer type.
*/
* @brief 32-bit integer type.
*/
typedef int32_t int32;
#endif
#if !defined(uint32)
/** @typedef uint32
* @brief 32-bit unsigned integer type.
*/
* @brief 32-bit unsigned integer type.
*/
typedef uint32_t uint32;
#endif
#if !defined(int8)
/** @typedef int8
* @brief 8-bit integer type.
*/
* @brief 8-bit integer type.
*/
typedef int8_t int8;
#endif
#if !defined(uint8)
/** @typedef uint8
* @brief 8-bit unsigned integer type.
*/
* @brief 8-bit unsigned integer type.
*/
typedef uint8_t uint8;
#endif
/** @typedef ByteArray
* @brief Type definition for a byte array (vector of chars).
*/
* @brief Type definition for a byte array (vector of chars).
*/
typedef std::vector<char> ByteArray;
/** @typedef Point2i
* @brief 2D coordinate point with integer precision.
*/
typedef cv::Point Point2i;
* @brief 2D coordinate point with integer precision.
*/
typedef inspirecv::Point2i Point2i;
/** @typedef Point2f
* @brief 2D coordinate point with float precision.
*/
typedef cv::Point2f Point2f;
* @brief 2D coordinate point with float precision.
*/
typedef inspirecv::Point2f Point2f;
/** @typedef PointsList2i
* @brief List of 2D coordinate points with integer precision.
*/
* @brief List of 2D coordinate points with integer precision.
*/
typedef std::vector<Point2i> PointsList2i;
/** @typedef PointsList2f
* @brief List of 2D coordinate points with float precision.
*/
* @brief List of 2D coordinate points with float precision.
*/
typedef std::vector<Point2f> PointsList2f;
/** @typedef Contours2i
* @brief Contours represented as a list of 2D integer points.
*/
* @brief Contours represented as a list of 2D integer points.
*/
typedef std::vector<PointsList2i> Contours2i;
/** @typedef Contours2f
* @brief Contours represented as a list of 2D float points.
*/
* @brief Contours represented as a list of 2D float points.
*/
typedef std::vector<PointsList2f> Contours2f;
/** @typedef Textures2i
* @brief Texture lines represented as integer contours.
*/
* @brief Texture lines represented as integer contours.
*/
typedef Contours2i Textures2i;
/** @typedef AnyTensorFp32
* @brief Generic tensor representation using a vector of floats.
*/
* @brief Generic tensor representation using a vector of floats.
*/
typedef std::vector<float> AnyTensorFp32;
/** @typedef Matrix
* @brief Generic matrix representation.
*/
typedef cv::Mat Matrix;
/** @typedef ImageBitmap
* @brief Image bitmap representation.
*/
typedef inspirecv::Image ImageBitmap;
/** @typedef Rectangle
* @brief Rectangle representation using integer values.
*/
typedef cv::Rect_<int> Rectangle;
* @brief Rectangle representation using integer values.
*/
typedef inspirecv::Rect<int> Rectangle;
/** @typedef Size
* @brief Size representation using integer values.
*/
typedef cv::Size_<int> Size;
* @brief Size representation using integer values.
*/
typedef inspirecv::Size<int> Size;
/** @typedef Embedded
* @brief Dense vector for feature embedding.
*/
* @brief Dense vector for feature embedding.
*/
typedef std::vector<float> Embedded;
/** @typedef EmbeddedList
* @brief List of dense vectors for feature embedding.
*/
* @brief List of dense vectors for feature embedding.
*/
typedef std::vector<Embedded> EmbeddedList;
/** @typedef String
* @brief String type definition.
*/
* @brief String type definition.
*/
typedef std::string String;
/** @typedef IndexList
* @brief List of indices.
*/
* @brief List of indices.
*/
typedef std::vector<int> IndexList;
/** @struct FaceLoc
* @brief Struct representing standardized face landmarks for detection.
*
* Contains coordinates for the face, detection score, and landmarks.
*/
* @brief Struct representing standardized face landmarks for detection.
*
* Contains coordinates for the face, detection score, and landmarks.
*/
typedef struct FaceLoc {
float x1;
float y1;
@@ -166,32 +167,32 @@ typedef struct FaceLoc {
} FaceLoc;
/** @typedef FaceLocList
* @brief List of FaceLoc structures.
*/
* @brief List of FaceLoc structures.
*/
typedef std::vector<FaceLoc> FaceLocList;
/** @struct FaceBasicData
* @brief Struct for basic face data.
*
* Contains the size of the data and a pointer to the data.
*/
* @brief Struct for basic face data.
*
* Contains the size of the data and a pointer to the data.
*/
typedef struct FaceBasicData {
int32_t dataSize;
void* data;
} FaceBasicData;
/** @struct FaceFeatureEntity
* @brief Struct for face feature data.
*
* Contains the size of the feature data and a pointer to the feature array.
*/
* @brief Struct for face feature data.
*
* Contains the size of the feature data and a pointer to the feature array.
*/
typedef struct FaceFeatureEntity {
int32_t dataSize;
float *data;
float* data;
} FaceFeaturePtr;
/** @} */
} // namespace inspire
#endif //HYPERFACE_DATATYPE_H
#endif // INSPIRE_FACE_DATATYPE_H

196
cpp-package/inspireface/cpp/inspireface/face_context.cpp → cpp-package/inspireface/cpp/inspireface/face_session.cpp
View File

@@ -1,20 +1,22 @@
//
// Created by Tunm-Air13 on 2023/9/7.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "face_context.h"
#include "Initialization_module/launch.h"
#include "face_session.h"
#include "initialization_module/launch.h"
#include <utility>
#include "log.h"
#include "herror.h"
#include "middleware/utils.h"
#include "recognition_module/dest_const.h"
namespace inspire {
FaceContext::FaceContext() = default;
FaceSession::FaceSession() = default;
int32_t FaceContext::Configuration(DetectMode detect_mode, int32_t max_detect_face, CustomPipelineParameter param,
int32_t detect_level_px, int32_t track_by_detect_mode_fps) {
int32_t FaceSession::Configuration(DetectModuleMode detect_mode, int32_t max_detect_face, CustomPipelineParameter param, int32_t detect_level_px,
int32_t track_by_detect_mode_fps) {
m_detect_mode_ = detect_mode;
m_max_detect_face_ = max_detect_face;
m_parameter_ = param;
@@ -25,26 +27,32 @@ int32_t FaceContext::Configuration(DetectMode detect_mode, int32_t max_detect_fa
return HERR_ARCHIVE_LOAD_FAILURE;
}
m_face_track_ = std::make_shared<FaceTrack>(m_detect_mode_, m_max_detect_face_, 20, 192, detect_level_px,
track_by_detect_mode_fps);
if (m_parameter_.enable_interaction_liveness) {
m_parameter_.enable_detect_mode_landmark = true;
}
m_face_track_ = std::make_shared<FaceTrackModule>(m_detect_mode_, m_max_detect_face_, 20, 192, detect_level_px, track_by_detect_mode_fps,
m_parameter_.enable_detect_mode_landmark);
m_face_track_->Configuration(INSPIRE_LAUNCH->getMArchive());
// SetDetectMode(m_detect_mode_);
m_face_recognition_ =
std::make_shared<FeatureExtraction>(INSPIRE_LAUNCH->getMArchive(), m_parameter_.enable_recognition);
m_face_recognition_ = std::make_shared<FeatureExtractionModule>(INSPIRE_LAUNCH->getMArchive(), m_parameter_.enable_recognition);
if (m_face_recognition_->QueryStatus() != HSUCCEED) {
return m_face_recognition_->QueryStatus();
}
m_face_pipeline_ =
std::make_shared<FacePipeline>(INSPIRE_LAUNCH->getMArchive(), param.enable_liveness, param.enable_mask_detect,
param.enable_face_attribute, param.enable_interaction_liveness);
m_face_pipeline_ = std::make_shared<FacePipelineModule>(INSPIRE_LAUNCH->getMArchive(), param.enable_liveness, param.enable_mask_detect,
param.enable_face_attribute, param.enable_interaction_liveness);
m_face_track_cost_ = std::make_shared<inspirecv::TimeSpend>("FaceTrack");
return HSUCCEED;
}
int32_t FaceContext::FaceDetectAndTrack(CameraStream& image) {
int32_t FaceSession::FaceDetectAndTrack(inspirecv::InspireImageProcess& process) {
std::lock_guard<std::mutex> lock(m_mtx_);
if (m_enable_track_cost_spend_) {
m_face_track_cost_->Start();
}
m_detect_cache_.clear();
m_face_basic_data_cache_.clear();
m_face_rects_cache_.clear();
@@ -70,12 +78,12 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream& image) {
if (m_face_track_ == nullptr) {
return HERR_SESS_TRACKER_FAILURE;
}
m_face_track_->UpdateStream(image);
m_face_track_->UpdateStream(process);
for (int i = 0; i < m_face_track_->trackingFace.size(); ++i) {
auto& face = m_face_track_->trackingFace[i];
HyperFaceData data = FaceObjectToHyperFaceData(face, i);
HyperFaceData data = FaceObjectInternalToHyperFaceData(face, i);
ByteArray byteArray;
auto ret = SerializeHyperFaceData(data, byteArray);
auto ret = RunSerializeHyperFaceData(data, byteArray);
if (ret != HSUCCEED) {
return HERR_INVALID_SERIALIZATION_FAILED;
}
@@ -103,33 +111,35 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream& image) {
basic.dataSize = m_detect_cache_[i].size();
basic.data = m_detect_cache_[i].data();
}
if (m_enable_track_cost_spend_) {
m_face_track_cost_->Stop();
}
// LOGD("Track COST: %f", m_face_track_->GetTrackTotalUseTime());
return HSUCCEED;
}
int32_t FaceContext::SetFaceDetectThreshold(float value) {
int32_t FaceSession::SetFaceDetectThreshold(float value) {
m_face_track_->SetDetectThreshold(value);
return HSUCCEED;
}
FaceObjectList& FaceContext::GetTrackingFaceList() {
FaceObjectInternalList& FaceSession::GetTrackingFaceList() {
return m_face_track_->trackingFace;
}
const std::shared_ptr<FeatureExtraction>& FaceContext::FaceRecognitionModule() {
const std::shared_ptr<FeatureExtractionModule>& FaceSession::FaceRecognitionModule() {
return m_face_recognition_;
}
const std::shared_ptr<FacePipeline>& FaceContext::FacePipelineModule() {
const std::shared_ptr<FacePipelineModule>& FaceSession::PipelineModule() {
return m_face_pipeline_;
}
const int32_t FaceContext::GetNumberOfFacesCurrentlyDetected() const {
const int32_t FaceSession::GetNumberOfFacesCurrentlyDetected() const {
return m_face_track_->trackingFace.size();
}
int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFaceData>& faces,
int32_t FaceSession::FacesProcess(inspirecv::InspireImageProcess& process, const std::vector<HyperFaceData>& faces,
const CustomPipelineParameter& param) {
std::lock_guard<std::mutex> lock(m_mtx_);
m_mask_results_cache_.resize(faces.size(), -1.0f);
@@ -148,7 +158,7 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
const auto& face = faces[i];
// RGB Liveness Detect
if (param.enable_liveness) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_RGB_LIVENESS);
auto ret = m_face_pipeline_->Process(process, face, PROCESS_RGB_LIVENESS);
if (ret != HSUCCEED) {
return ret;
}
@@ -156,7 +166,7 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
}
// Mask detection
if (param.enable_mask_detect) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_MASK);
auto ret = m_face_pipeline_->Process(process, face, PROCESS_MASK);
if (ret != HSUCCEED) {
return ret;
}
@@ -164,7 +174,7 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
}
// Face attribute prediction
if (param.enable_face_attribute) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_ATTRIBUTE);
auto ret = m_face_pipeline_->Process(process, face, PROCESS_ATTRIBUTE);
if (ret != HSUCCEED) {
return ret;
}
@@ -175,7 +185,7 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
// Face interaction
if (param.enable_interaction_liveness) {
auto ret = m_face_pipeline_->Process(image, face, PROCESS_INTERACTION);
auto ret = m_face_pipeline_->Process(process, face, PROCESS_INTERACTION);
if (ret != HSUCCEED) {
return ret;
}
@@ -188,10 +198,8 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
if (idx < m_face_track_->trackingFace.size()) {
auto& target = m_face_track_->trackingFace[idx];
if (target.GetTrackingId() == face.trackId) {
auto new_eye_left =
EmaFilter(m_face_pipeline_->eyesStatusCache[0], target.left_eye_status_, 8, 0.2f);
auto new_eye_right =
EmaFilter(m_face_pipeline_->eyesStatusCache[1], target.right_eye_status_, 8, 0.2f);
auto new_eye_left = EmaFilter(m_face_pipeline_->eyesStatusCache[0], target.left_eye_status_, 8, 0.2f);
auto new_eye_right = EmaFilter(m_face_pipeline_->eyesStatusCache[1], target.right_eye_status_, 8, 0.2f);
if (face.trackState > 1) {
// The filtered value can be obtained only in the tracking state
m_react_left_eye_results_cache_[i] = new_eye_left;
@@ -220,120 +228,141 @@ int32_t FaceContext::FacesProcess(CameraStream& image, const std::vector<HyperFa
return 0;
}
const std::vector<ByteArray>& FaceContext::GetDetectCache() const {
const std::vector<ByteArray>& FaceSession::GetDetectCache() const {
return m_detect_cache_;
}
const std::vector<FaceBasicData>& FaceContext::GetFaceBasicDataCache() const {
const std::vector<FaceBasicData>& FaceSession::GetFaceBasicDataCache() const {
return m_face_basic_data_cache_;
}
const std::vector<FaceRect>& FaceContext::GetFaceRectsCache() const {
const std::vector<FaceRect>& FaceSession::GetFaceRectsCache() const {
return m_face_rects_cache_;
}
const std::vector<int32_t>& FaceContext::GetTrackIDCache() const {
const std::vector<int32_t>& FaceSession::GetTrackIDCache() const {
return m_track_id_cache_;
}
const std::vector<float>& FaceContext::GetRollResultsCache() const {
const std::vector<float>& FaceSession::GetRollResultsCache() const {
return m_roll_results_cache_;
}
const std::vector<float>& FaceContext::GetYawResultsCache() const {
const std::vector<float>& FaceSession::GetYawResultsCache() const {
return m_yaw_results_cache_;
}
const std::vector<float>& FaceContext::GetPitchResultsCache() const {
const std::vector<float>& FaceSession::GetPitchResultsCache() const {
return m_pitch_results_cache_;
}
const std::vector<FacePoseQualityResult>& FaceContext::GetQualityResultsCache() const {
const std::vector<FacePoseQualityAdaptResult>& FaceSession::GetQualityResultsCache() const {
return m_quality_results_cache_;
}
const std::vector<float>& FaceContext::GetMaskResultsCache() const {
const std::vector<float>& FaceSession::GetMaskResultsCache() const {
return m_mask_results_cache_;
}
const std::vector<float>& FaceContext::GetRgbLivenessResultsCache() const {
const std::vector<float>& FaceSession::GetRgbLivenessResultsCache() const {
return m_rgb_liveness_results_cache_;
}
const std::vector<float>& FaceContext::GetFaceQualityScoresResultsCache() const {
const std::vector<float>& FaceSession::GetFaceQualityScoresResultsCache() const {
return m_quality_score_results_cache_;
}
const std::vector<float>& FaceContext::GetFaceInteractionLeftEyeStatusCache() const {
const std::vector<float>& FaceSession::GetFaceInteractionLeftEyeStatusCache() const {
return m_react_left_eye_results_cache_;
}
const std::vector<float>& FaceContext::GetFaceInteractionRightEyeStatusCache() const {
const std::vector<float>& FaceSession::GetFaceInteractionRightEyeStatusCache() const {
return m_react_right_eye_results_cache_;
}
const Embedded& FaceContext::GetFaceFeatureCache() const {
const Embedded& FaceSession::GetFaceFeatureCache() const {
return m_face_feature_cache_;
}
const std::vector<float>& FaceContext::GetDetConfidenceCache() const {
const std::vector<float>& FaceSession::GetDetConfidenceCache() const {
return m_det_confidence_cache_;
}
const std::vector<int>& FaceContext::GetFaceRaceResultsCache() const {
const float FaceSession::GetFaceFeatureNormCache() const {
return m_face_feature_norm_;
}
const std::vector<int>& FaceSession::GetFaceRaceResultsCache() const {
return m_attribute_race_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceGenderResultsCache() const {
const std::vector<int>& FaceSession::GetFaceGenderResultsCache() const {
return m_attribute_gender_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceAgeBracketResultsCache() const {
const std::vector<int>& FaceSession::GetFaceAgeBracketResultsCache() const {
return m_attribute_age_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceNormalAactionsResultCache() const {
const std::vector<int>& FaceSession::GetFaceNormalAactionsResultCache() const {
return m_action_normal_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceJawOpenAactionsResultCache() const {
const std::vector<int>& FaceSession::GetFaceJawOpenAactionsResultCache() const {
return m_action_jaw_open_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceBlinkAactionsResultCache() const {
const std::vector<int>& FaceSession::GetFaceBlinkAactionsResultCache() const {
return m_action_blink_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceShakeAactionsResultCache() const {
const std::vector<int>& FaceSession::GetFaceShakeAactionsResultCache() const {
return m_action_shake_results_cache_;
}
const std::vector<int>& FaceContext::GetFaceRaiseHeadAactionsResultCache() const {
const std::vector<int>& FaceSession::GetFaceRaiseHeadAactionsResultCache() const {
return m_action_raise_head_results_cache_;
}
int32_t FaceContext::FaceFeatureExtract(CameraStream& image, FaceBasicData& data) {
int32_t FaceSession::FaceFeatureExtract(inspirecv::InspireImageProcess& process, FaceBasicData& data) {
std::lock_guard<std::mutex> lock(m_mtx_);
int32_t ret;
HyperFaceData face = {0};
ret = DeserializeHyperFaceData((char*)data.data, data.dataSize, face);
ret = RunDeserializeHyperFaceData((char*)data.data, data.dataSize, face);
if (ret != HSUCCEED) {
return ret;
}
m_face_feature_cache_.clear();
ret = m_face_recognition_->FaceExtract(image, face, m_face_feature_cache_);
ret = m_face_recognition_->FaceExtract(process, face, m_face_feature_cache_, m_face_feature_norm_);
return ret;
}
const CustomPipelineParameter& FaceContext::getMParameter() const {
int32_t FaceSession::FaceGetFaceAlignmentImage(inspirecv::InspireImageProcess& process, FaceBasicData& data, inspirecv::Image& image) {
std::lock_guard<std::mutex> lock(m_mtx_);
int32_t ret;
HyperFaceData face = {0};
ret = RunDeserializeHyperFaceData((char*)data.data, data.dataSize, face);
if (ret != HSUCCEED) {
return ret;
}
std::vector<inspirecv::Point2f> pointsFive;
for (const auto& p : face.keyPoints) {
pointsFive.push_back(inspirecv::Point2f(p.x, p.y));
}
auto trans = inspirecv::SimilarityTransformEstimateUmeyama(SIMILARITY_TRANSFORM_DEST, pointsFive);
image = process.ExecuteImageAffineProcessing(trans, FACE_CROP_SIZE, FACE_CROP_SIZE);
return ret;
}
const CustomPipelineParameter& FaceSession::getMParameter() const {
return m_parameter_;
}
int32_t FaceContext::FaceQualityDetect(FaceBasicData& data, float& result) {
int32_t FaceSession::FaceQualityDetect(FaceBasicData& data, float& result) {
int32_t ret;
HyperFaceData face = {0};
ret = DeserializeHyperFaceData((char*)data.data, data.dataSize, face);
ret = RunDeserializeHyperFaceData((char*)data.data, data.dataSize, face);
// PrintHyperFaceData(face);
if (ret != HSUCCEED) {
return ret;
@@ -348,9 +377,9 @@ int32_t FaceContext::FaceQualityDetect(FaceBasicData& data, float& result) {
return ret;
}
int32_t FaceContext::SetDetectMode(DetectMode mode) {
int32_t FaceSession::SetDetectMode(DetectModuleMode mode) {
m_detect_mode_ = mode;
if (m_detect_mode_ == DetectMode::DETECT_MODE_ALWAYS_DETECT) {
if (m_detect_mode_ == DetectModuleMode::DETECT_MODE_ALWAYS_DETECT) {
m_always_detect_ = true;
} else {
m_always_detect_ = false;
@@ -358,14 +387,45 @@ int32_t FaceContext::SetDetectMode(DetectMode mode) {
return HSUCCEED;
}
int32_t FaceContext::SetTrackPreviewSize(const int32_t preview_size) {
bool FaceSession::IsDetectModeLandmark() const {
return m_face_track_->IsDetectModeLandmark();
}
int32_t FaceSession::SetTrackPreviewSize(const int32_t preview_size) {
m_face_track_->SetTrackPreviewSize(preview_size);
return HSUCCEED;
}
int32_t FaceContext::SetTrackFaceMinimumSize(int32_t minSize) {
int32_t FaceSession::SetTrackFaceMinimumSize(int32_t minSize) {
m_face_track_->SetMinimumFacePxSize(minSize);
return HSUCCEED;
}
} // namespace inspire
int32_t FaceSession::SetTrackModeSmoothRatio(float value) {
m_face_track_->SetTrackModeSmoothRatio(value);
return HSUCCEED;
}
int32_t FaceSession::SetTrackModeNumSmoothCacheFrame(int value) {
m_face_track_->SetTrackModeNumSmoothCacheFrame(value);
return HSUCCEED;
}
int32_t FaceSession::SetTrackModeDetectInterval(int value) {
m_face_track_->SetTrackModeDetectInterval(value);
return HSUCCEED;
}
int32_t FaceSession::SetEnableTrackCostSpend(int value) {
m_enable_track_cost_spend_ = value;
m_face_track_cost_->Reset();
return HSUCCEED;
}
void FaceSession::PrintTrackCostSpend() {
if (m_enable_track_cost_spend_) {
INSPIRE_LOGI("%s", m_face_track_cost_->Report().c_str());
}
}
} // namespace inspire

161
cpp-package/inspireface/cpp/inspireface/face_context.h → cpp-package/inspireface/cpp/inspireface/face_session.h
View File

@@ -1,29 +1,21 @@
//
// Created by Tunm-Air13 on 2023/9/7.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#include <vector>
#ifndef HYPERFACEREPO_FACE_CONTEXT_H
#define HYPERFACEREPO_FACE_CONTEXT_H
/**
* @file face_context.h
* @brief Face context handling for HyperFaceRepo project.
* Includes definitions for face detection, tracking, and feature extraction.
*/
#ifndef INSPIRE_FACE_CONTEXT_H
#define INSPIRE_FACE_CONTEXT_H
#include <memory>
#include "track_module/face_track.h"
#include <inspirecv/inspirecv.h>
#include "data_type.h"
#include "pipeline_module/face_pipeline.h"
#include "recognition_module/face_feature_extraction.h"
#include "track_module/face_track_module.h"
#include "pipeline_module/face_pipeline_module.h"
#include "middleware/model_archive/inspire_archive.h"
/**
* @def DB_FILE_NAME
* @brief Default database file name used in the FaceContext.
*/
#define DB_FILE_NAME ".E63520A95DD5B3892C56DA38C3B28E551D8173FD"
#include "recognition_module/face_feature_extraction_module.h"
#include "middleware/inspirecv_image_process.h"
#include "common/face_data/face_serialize_tools.h"
namespace inspire {
@@ -41,6 +33,7 @@ typedef struct CustomPipelineParameter {
bool enable_face_attribute = false; ///< Enable face attribute prediction feature
bool enable_face_quality = false; ///< Enable face quality assessment feature
bool enable_interaction_liveness = false; ///< Enable interactive liveness detection feature
bool enable_detect_mode_landmark = false; ///< Enable landmark detection in detection mode
} ContextCustomParameter;
@@ -52,12 +45,12 @@ typedef struct CustomPipelineParameter {
* and handle other face-related features. Integrates with various modules such as FaceTrack, FaceRecognition, and
* FacePipeline.
*/
class INSPIRE_API FaceContext {
class INSPIRE_API FaceSession {
public:
/**
* @brief Constructor for the FaceContext class.
*/
explicit FaceContext();
explicit FaceSession();
/**
* @brief Configures the face context with given parameters.
@@ -67,7 +60,7 @@ public:
* @param param Custom parameters for the face pipeline.
* @return int32_t Returns 0 on success, non-zero for any error.
*/
int32_t Configuration(DetectMode detect_mode, int32_t max_detect_face, CustomPipelineParameter param, int32_t detect_level_px = -1,
int32_t Configuration(DetectModuleMode detect_mode, int32_t max_detect_face, CustomPipelineParameter param, int32_t detect_level_px = -1,
int32_t track_by_detect_mode_fps = -1);
/**
@@ -75,7 +68,7 @@ public:
* @param image The camera stream to process for face detection and tracking.
* @return int32_t Returns the number of faces detected and tracked.
*/// Method for face detection and tracking
int32_t FaceDetectAndTrack(CameraStream& image);
int32_t FaceDetectAndTrack(inspirecv::InspireImageProcess& process);
/**
* @brief Set the threshold of face detection function, which only acts on the detection model
@@ -88,7 +81,7 @@ public:
* @brief Retrieves the list of currently tracked faces.
* @return FaceObjectList A list of face objects currently being tracked.
*/
FaceObjectList& GetTrackingFaceList();
FaceObjectInternalList& GetTrackingFaceList();
/**
* @brief Processes faces using the provided pipeline parameters.
@@ -97,19 +90,19 @@ public:
* @param param Custom pipeline parameters.
* @return int32_t Status code of the processing.
*/
int32_t FacesProcess(CameraStream& image, const std::vector<HyperFaceData>& faces, const CustomPipelineParameter& param);
int32_t FacesProcess(inspirecv::InspireImageProcess& process, const std::vector<HyperFaceData>& faces, const CustomPipelineParameter& param);
/**
* @brief Retrieves the face recognition module.
* @return std::shared_ptr<FaceRecognition> Shared pointer to the FaceRecognition module.
*/
const std::shared_ptr<FeatureExtraction>& FaceRecognitionModule();
const std::shared_ptr<FeatureExtractionModule>& FaceRecognitionModule();
/**
* @brief Retrieves the face pipeline module.
* @return std::shared_ptr<FacePipeline> Shared pointer to the FacePipeline module.
*/
const std::shared_ptr<FacePipeline>& FacePipelineModule();
const std::shared_ptr<FacePipelineModule>& PipelineModule();
/**
* @brief Gets the number of faces currently detected.
@@ -123,7 +116,16 @@ public:
* @param data FaceBasicData to store extracted features.
* @return int32_t Status code of the feature extraction.
*/
int32_t FaceFeatureExtract(CameraStream& image, FaceBasicData& data);
int32_t FaceFeatureExtract(inspirecv::InspireImageProcess& process, FaceBasicData& data);
/**
* @brief Gets the face alignment image.
* @param process The image process object.
* @param data The face basic data.
* @param image The output image.
* @return int32_t The status code of the operation.
*/
int32_t FaceGetFaceAlignmentImage(inspirecv::InspireImageProcess& process, FaceBasicData& data, inspirecv::Image& image);
/**
* @brief Retrieves the custom pipeline parameters.
@@ -158,7 +160,13 @@ public:
* @param mode You can select mode for track or detect.
* @return int32_t Status code of the operation.
* */
int32_t SetDetectMode(DetectMode mode);
int32_t SetDetectMode(DetectModuleMode mode);
/**
* @brief Check if landmark detection is enabled in detection mode.
* @return True if landmark detection is enabled, false otherwise.
*/
bool IsDetectModeLandmark() const;
public:
// Accessor methods for various cached data
@@ -208,7 +216,7 @@ public:
* @brief Gets the cache of face pose quality results.
* @return A const reference to a vector of FacePoseQualityResult objects.
*/
const std::vector<FacePoseQualityResult>& GetQualityResultsCache() const;
const std::vector<FacePoseQualityAdaptResult>& GetQualityResultsCache() const;
/**
* @brief Gets the cache of mask detection results.
@@ -300,33 +308,72 @@ public:
*/
const std::vector<float>& GetDetConfidenceCache() const;
/**
* @brief Gets the cache of face feature norm.
* @return A const reference to a float containing face feature norm.
*/
const float GetFaceFeatureNormCache() const;
/**
* @brief Set the track mode smooth ratio
* @param value The smooth ratio value
* @return int32_t Status code of the operation.
* */
int32_t SetTrackModeSmoothRatio(float value);
/**
* @brief Set the track mode num smooth cache frame
* @param value The num smooth cache frame value
* @return int32_t Status code of the operation.
* */
int32_t SetTrackModeNumSmoothCacheFrame(int value);
/**
* @brief Set the track model detect interval
* @param value The detect interval value
* @return int32_t Status code of the operation.
* */
int32_t SetTrackModeDetectInterval(int value);
/**
* @brief Set the enable cost spend
* @param value The enable cost spend value
* @return int32_t Status code of the operation.
* */
int32_t SetEnableTrackCostSpend(int value);
/**
* @brief Print the cost spend
* */
void PrintTrackCostSpend();
private:
// Private member variables
CustomPipelineParameter m_parameter_; ///< Stores custom parameters for the pipeline
int32_t m_max_detect_face_{}; ///< Maximum number of faces that can be detected
DetectMode m_detect_mode_; ///< Current detection mode (image or video)
DetectModuleMode m_detect_mode_; ///< Current detection mode (image or video)
bool m_always_detect_{}; ///< Flag to determine if detection should always occur
std::shared_ptr<FaceTrack> m_face_track_; ///< Shared pointer to the FaceTrack object
std::shared_ptr<FeatureExtraction> m_face_recognition_; ///< Shared pointer to the FaceRecognition object
std::shared_ptr<FacePipeline> m_face_pipeline_; ///< Shared pointer to the FacePipeline object
std::shared_ptr<FaceTrackModule> m_face_track_; ///< Shared pointer to the FaceTrack object
std::shared_ptr<FeatureExtractionModule> m_face_recognition_; ///< Shared pointer to the FaceRecognition object
std::shared_ptr<FacePipelineModule> m_face_pipeline_; ///< Shared pointer to the FacePipeline object
private:
// Cache data
std::vector<ByteArray> m_detect_cache_; ///< Cache for storing serialized detected face data
std::vector<FaceBasicData> m_face_basic_data_cache_; ///< Cache for basic face data extracted from detection
std::vector<FaceRect> m_face_rects_cache_; ///< Cache for face rectangle data from detection
std::vector<int32_t> m_track_id_cache_; ///< Cache for tracking IDs of detected faces
std::vector<float> m_det_confidence_cache_; ///< Cache for face detection confidence of detected faces
std::vector<float> m_roll_results_cache_; ///< Cache for storing roll results from face pose estimation
std::vector<float> m_yaw_results_cache_; ///< Cache for storing yaw results from face pose estimation
std::vector<float> m_pitch_results_cache_; ///< Cache for storing pitch results from face pose estimation
std::vector<FacePoseQualityResult> m_quality_results_cache_; ///< Cache for face pose quality results
std::vector<float> m_mask_results_cache_; ///< Cache for mask detection results
std::vector<float> m_rgb_liveness_results_cache_; ///< Cache for RGB liveness detection results
std::vector<float> m_quality_score_results_cache_; ///< Cache for RGB face quality score results
std::vector<float> m_react_left_eye_results_cache_; ///< Cache for Left eye state in face interaction
std::vector<float> m_react_right_eye_results_cache_; ///< Cache for Right eye state in face interaction
std::vector<ByteArray> m_detect_cache_; ///< Cache for storing serialized detected face data
std::vector<FaceBasicData> m_face_basic_data_cache_; ///< Cache for basic face data extracted from detection
std::vector<FaceRect> m_face_rects_cache_; ///< Cache for face rectangle data from detection
std::vector<int32_t> m_track_id_cache_; ///< Cache for tracking IDs of detected faces
std::vector<float> m_det_confidence_cache_; ///< Cache for face detection confidence of detected faces
std::vector<float> m_roll_results_cache_; ///< Cache for storing roll results from face pose estimation
std::vector<float> m_yaw_results_cache_; ///< Cache for storing yaw results from face pose estimation
std::vector<float> m_pitch_results_cache_; ///< Cache for storing pitch results from face pose estimation
std::vector<FacePoseQualityAdaptResult> m_quality_results_cache_; ///< Cache for face pose quality results
std::vector<float> m_mask_results_cache_; ///< Cache for mask detection results
std::vector<float> m_rgb_liveness_results_cache_; ///< Cache for RGB liveness detection results
std::vector<float> m_quality_score_results_cache_; ///< Cache for RGB face quality score results
std::vector<float> m_react_left_eye_results_cache_; ///< Cache for Left eye state in face interaction
std::vector<float> m_react_right_eye_results_cache_; ///< Cache for Right eye state in face interaction
std::vector<int> m_action_normal_results_cache_; ///< Cache for normal action in face interaction
std::vector<int> m_action_shake_results_cache_; ///< Cache for shake action in face interaction
@@ -334,14 +381,20 @@ private:
std::vector<int> m_action_jaw_open_results_cache_; ///< Cache for jaw open action in face interaction
std::vector<int> m_action_raise_head_results_cache_; ///< Cache for raise head action in face interaction
std::vector<int> m_attribute_race_results_cache_;
std::vector<int> m_attribute_gender_results_cache_;
std::vector<int> m_attribute_age_results_cache_;
Embedded m_face_feature_cache_; ///< Cache for current face feature data
std::vector<int> m_attribute_race_results_cache_; ///< Cache for face attribute race results
std::vector<int> m_attribute_gender_results_cache_; ///< Cache for face attribute gender results
std::vector<int> m_attribute_age_results_cache_; ///< Cache for face attribute age results
Embedded m_face_feature_cache_; ///< Cache for current face feature data
float m_face_feature_norm_; ///< Cache for face feature norm
std::mutex m_mtx_; ///< Mutex for thread safety.
// cost spend
std::shared_ptr<inspirecv::TimeSpend> m_face_track_cost_;
int m_enable_track_cost_spend_ = 0;
};
} // namespace inspire
#endif // HYPERFACEREPO_FACE_CONTEXT_H
#endif // INSPIRE_FACE_CONTEXT_H

368
cpp-package/inspireface/cpp/inspireface/feature_hub/embedding_db/embedding_db.cpp Normal file
View File

@@ -0,0 +1,368 @@
#include "embedding_db.h"
#include "sqlite-vec.h"
#include "isf_check.h"
#include <algorithm>
namespace inspire {
std::unique_ptr<EmbeddingDB> EmbeddingDB::instance_ = nullptr;
std::mutex EmbeddingDB::instanceMutex_;
EmbeddingDB &EmbeddingDB::GetInstance() {
std::lock_guard<std::mutex> lock(instanceMutex_);
INSPIREFACE_CHECK_MSG(instance_, "EmbeddingDB not initialized. Call Init() first.");
return *instance_;
}
void EmbeddingDB::Init(const std::string &dbPath, size_t vectorDim, IdMode idMode) {
std::lock_guard<std::mutex> lock(instanceMutex_);
INSPIREFACE_CHECK_MSG(!instance_, "EmbeddingDB already initialized");
instance_.reset(new EmbeddingDB(dbPath, vectorDim, "cosine", idMode));
}
EmbeddingDB::EmbeddingDB(const std::string &dbPath, size_t vectorDim, const std::string &distanceMetric, IdMode idMode)
: vectorDim_(vectorDim), tableName_("vec_items"), idMode_(idMode) {
int rc = sqlite3_auto_extension((void (*)())sqlite3_vec_init);
CheckSQLiteError(rc, nullptr);
// Open database
rc = sqlite3_open(dbPath.c_str(), &db_);
CheckSQLiteError(rc, db_);
// Create vector table
std::string createTableSQL = "CREATE VIRTUAL TABLE IF NOT EXISTS " + tableName_ + " USING vec0(embedding float[" + std::to_string(vectorDim_) +
"] distance_metric=" + distanceMetric + ")";
ExecuteSQL(createTableSQL);
initialized_ = true;
}
EmbeddingDB::~EmbeddingDB() {
if (db_) {
sqlite3_close(db_);
}
}
bool EmbeddingDB::InsertVector(const std::vector<float> &vector, int64_t &allocId) {
std::lock_guard<std::mutex> lock(dbMutex_);
return InsertVector(0, vector, allocId); // In auto-increment mode, the passed ID is ignored
}
bool EmbeddingDB::InsertVector(int64_t id, const std::vector<float> &vector, int64_t &allocId) {
CheckVectorDimension(vector);
sqlite3_stmt *stmt;
std::string sql;
if (idMode_ == IdMode::AUTO_INCREMENT) {
sql = "INSERT INTO " + tableName_ + "(embedding) VALUES (?)";
} else {
sql = "INSERT INTO " + tableName_ + "(rowid, embedding) VALUES (?, ?)";
}
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
// CheckSQLiteError(rc, db_);
if (rc != SQLITE_OK) {
INSPIRE_LOGE("Failed to prepare statement: %s", sqlite3_errmsg(db_));
sqlite3_finalize(stmt);
return false;
}
if (idMode_ == IdMode::AUTO_INCREMENT) {
sqlite3_bind_blob(stmt, 1, vector.data(), vector.size() * sizeof(float), SQLITE_STATIC);
} else {
sqlite3_bind_int64(stmt, 1, id);
sqlite3_bind_blob(stmt, 2, vector.data(), vector.size() * sizeof(float), SQLITE_STATIC);
}
rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
if (rc != SQLITE_DONE) {
INSPIRE_LOGE("Failed to insert vector: %s", sqlite3_errmsg(db_));
return false;
}
// CheckSQLiteError(rc == SQLITE_DONE ? SQLITE_OK : rc, db_);
allocId = idMode_ == IdMode::AUTO_INCREMENT ? GetLastInsertRowId() : id;
return true;
}
std::vector<float> EmbeddingDB::GetVector(int64_t id) const {
std::lock_guard<std::mutex> lock(dbMutex_);
sqlite3_stmt *stmt;
std::string sql = "SELECT embedding FROM " + tableName_ + " WHERE rowid = ?";
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
sqlite3_bind_int64(stmt, 1, id);
rc = sqlite3_step(stmt);
if (rc != SQLITE_ROW) {
sqlite3_finalize(stmt);
// throw std::runtime_error("Vector with id " + std::to_string(id) + " not found");
return {};
}
const float *blob_data = static_cast<const float *>(sqlite3_column_blob(stmt, 0));
size_t blob_size = sqlite3_column_bytes(stmt, 0) / sizeof(float);
std::vector<float> result(blob_data, blob_data + blob_size);
sqlite3_finalize(stmt);
return result;
}
std::vector<int64_t> EmbeddingDB::BatchInsertVectors(const std::vector<VectorData> &vectors) {
ExecuteSQL("BEGIN");
std::vector<int64_t> insertedIds;
insertedIds.reserve(vectors.size());
try {
for (const auto &data : vectors) {
int64_t id = 0;
bool ret = InsertVector(data.id, data.vector, id);
if (!ret) {
throw std::runtime_error("Failed to insert vector");
}
insertedIds.push_back(id);
}
ExecuteSQL("COMMIT");
} catch (...) {
ExecuteSQL("ROLLBACK");
throw;
}
return insertedIds;
}
std::vector<int64_t> EmbeddingDB::BatchInsertVectors(const std::vector<std::vector<float>> &vectors) {
ExecuteSQL("BEGIN");
std::vector<int64_t> insertedIds;
insertedIds.reserve(vectors.size());
try {
for (const auto &vector : vectors) {
int64_t id = 0;
bool ret = InsertVector(0, vector, id);
if (!ret) {
throw std::runtime_error("Failed to insert vector");
}
insertedIds.push_back(id);
}
ExecuteSQL("COMMIT");
} catch (...) {
ExecuteSQL("ROLLBACK");
throw;
}
return insertedIds;
}
int64_t EmbeddingDB::GetLastInsertRowId() const {
return sqlite3_last_insert_rowid(db_);
}
void EmbeddingDB::UpdateVector(int64_t id, const std::vector<float> &newVector) {
CheckVectorDimension(newVector);
sqlite3_stmt *stmt;
std::string sql = "UPDATE " + tableName_ + " SET embedding = ? WHERE rowid = ?";
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
sqlite3_bind_blob(stmt, 1, newVector.data(), newVector.size() * sizeof(float), SQLITE_STATIC);
sqlite3_bind_int64(stmt, 2, id);
rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
INSPIREFACE_CHECK_MSG(rc == SQLITE_DONE, "Failed to update vector");
if (sqlite3_changes(db_) == 0) {
INSPIRE_LOGF("Vector with id %ld not found", id);
}
}
void EmbeddingDB::DeleteVector(int64_t id) {
sqlite3_stmt *stmt;
std::string sql = "DELETE FROM " + tableName_ + " WHERE rowid = ?";
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
sqlite3_bind_int64(stmt, 1, id);
rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
CheckSQLiteError(rc == SQLITE_DONE ? SQLITE_OK : rc, db_);
}
std::vector<FaceSearchResult> EmbeddingDB::SearchSimilarVectors(const std::vector<float> &queryVector, size_t top_k, float keep_similar_threshold,
bool return_feature) {
std::lock_guard<std::mutex> lock(dbMutex_);
CheckVectorDimension(queryVector);
sqlite3_stmt *stmt;
std::string sql;
if (return_feature) {
sql =
"SELECT rowid, embedding, 1.0 - distance as similarity "
"FROM " +
tableName_ +
" "
"WHERE embedding MATCH ? "
"ORDER BY distance "
"LIMIT ?";
} else {
sql =
"SELECT rowid, 1.0 - distance as similarity "
"FROM " +
tableName_ +
" "
"WHERE embedding MATCH ? "
"ORDER BY distance "
"LIMIT ?";
}
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
sqlite3_bind_blob(stmt, 1, queryVector.data(), queryVector.size() * sizeof(float), SQLITE_STATIC);
sqlite3_bind_int64(stmt, 2, top_k);
std::vector<FaceSearchResult> results;
while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
FaceSearchResult result;
result.id = sqlite3_column_int64(stmt, 0);
if (return_feature) {
const float *blob_data = static_cast<const float *>(sqlite3_column_blob(stmt, 1));
size_t blob_size = sqlite3_column_bytes(stmt, 1) / sizeof(float);
result.feature.assign(blob_data, blob_data + blob_size);
result.similarity = sqlite3_column_double(stmt, 2);
} else {
result.similarity = sqlite3_column_double(stmt, 1);
}
results.push_back(result);
}
sqlite3_finalize(stmt);
CheckSQLiteError(rc == SQLITE_DONE ? SQLITE_OK : rc, db_);
// Filter results whose similarity is below the threshold
results.erase(std::remove_if(results.begin(), results.end(),
[keep_similar_threshold](const FaceSearchResult &result) { return result.similarity < keep_similar_threshold; }),
results.end());
return results;
}
int64_t EmbeddingDB::GetVectorCount() const {
std::lock_guard<std::mutex> lock(dbMutex_);
sqlite3_stmt *stmt;
std::string sql = "SELECT COUNT(*) FROM " + tableName_;
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
rc = sqlite3_step(stmt);
CheckSQLiteError(rc == SQLITE_ROW ? SQLITE_OK : rc, db_);
int64_t count = sqlite3_column_int64(stmt, 0);
sqlite3_finalize(stmt);
return count;
}
void EmbeddingDB::CheckVectorDimension(const std::vector<float> &vector) const {
INSPIREFACE_CHECK_MSG(vector.size() == vectorDim_,
("Vector dimension mismatch. Expected: " + std::to_string(vectorDim_) + ", Got: " + std::to_string(vector.size())).c_str());
}
void EmbeddingDB::ExecuteSQL(const std::string &sql) {
std::lock_guard<std::mutex> lock(dbMutex_);
char *errMsg = nullptr;
int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &errMsg);
if (errMsg) {
std::string error = errMsg;
sqlite3_free(errMsg);
INSPIREFACE_CHECK_MSG(false, ("SQL error: " + error).c_str());
}
CheckSQLiteError(rc, db_);
}
void EmbeddingDB::CheckSQLiteError(int rc, sqlite3 *db) {
std::string error = db ? sqlite3_errmsg(db) : "SQLite error";
INSPIREFACE_CHECK_MSG(rc == SQLITE_OK, error.c_str());
}
void EmbeddingDB::ShowTable() {
if (!initialized_) {
INSPIRE_LOGE("EmbeddingDB is not initialized");
return;
}
std::lock_guard<std::mutex> lock(dbMutex_);
sqlite3_stmt *stmt;
std::string sql = "SELECT rowid, embedding FROM " + tableName_;
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
// Print header
#ifdef __ANDROID__
__android_log_print(ANDROID_LOG_INFO, "EmbeddingDB", "=== Table Content ===");
__android_log_print(ANDROID_LOG_INFO, "EmbeddingDB", "ID | Vector (first 5 elements)");
__android_log_print(ANDROID_LOG_INFO, "EmbeddingDB", "------------------------");
#else
printf("=== Table Content ===\n");
printf("ID | Vector (first 5 elements)\n");
printf("------------------------\n");
#endif
while (sqlite3_step(stmt) == SQLITE_ROW) {
int64_t id = sqlite3_column_int64(stmt, 0);
const float *vector_data = static_cast<const float *>(sqlite3_column_blob(stmt, 1));
size_t vector_size = std::min(size_t(5), sqlite3_column_bytes(stmt, 1) / sizeof(float));
std::string vector_str;
for (size_t i = 0; i < vector_size; ++i) {
vector_str += std::to_string(vector_data[i]);
if (i < vector_size - 1)
vector_str += ", ";
}
vector_str += "...";
#ifdef __ANDROID__
__android_log_print(ANDROID_LOG_INFO, "EmbeddingDB", "%lld | %s", id, vector_str.c_str());
#else
printf("%lld | %s\n", id, vector_str.c_str());
#endif
}
sqlite3_finalize(stmt);
}
std::vector<int64_t> EmbeddingDB::GetAllIds() {
if (!initialized_) {
INSPIRE_LOGE("EmbeddingDB is not initialized");
return {};
}
std::lock_guard<std::mutex> lock(dbMutex_);
std::vector<int64_t> ids;
sqlite3_stmt *stmt;
std::string sql = "SELECT rowid FROM " + tableName_;
int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr);
CheckSQLiteError(rc, db_);
while (sqlite3_step(stmt) == SQLITE_ROW) {
ids.push_back(sqlite3_column_int64(stmt, 0));
}
sqlite3_finalize(stmt);
return ids;
}
} // namespace inspire

131
cpp-package/inspireface/cpp/inspireface/feature_hub/embedding_db/embedding_db.h Normal file
View File

@@ -0,0 +1,131 @@
#ifndef INSPIRE_EMBEDDING_DB_H
#define INSPIRE_EMBEDDING_DB_H
#ifndef SQLITE_CORE
#define SQLITE_CORE
#endif
#ifndef SQLITE_VEC_STATIC
#define SQLITE_VEC_STATIC
#endif
#ifndef SQLITE_VEC_ENABLE_AVX
#define SQLITE_VEC_ENABLE_AVX
#endif
#include <sqlite3.h>
#include <vector>
#include <string>
#include <memory>
#include <stdexcept>
#include <mutex>
#define EMBEDDING_DB inspire::EmbeddingDB
namespace inspire {
// Search for most similar vectors
struct FaceSearchResult {
int64_t id;
double similarity;
std::vector<float> feature;
};
// Vector data structure
struct VectorData {
int64_t id; // This field is ignored in auto-increment mode
std::vector<float> vector;
};
// ID mode enumeration
enum class IdMode {
AUTO_INCREMENT = 0, // Auto-incrementing ID
MANUAL, // Manually specify ID
};
class EmbeddingDB {
public:
~EmbeddingDB();
static EmbeddingDB &GetInstance();
static void Init(const std::string &dbPath = ":memory:", size_t vectorDim = 512, IdMode idMode = IdMode::AUTO_INCREMENT);
// Delete copy and move operations
EmbeddingDB(const EmbeddingDB &) = delete;
EmbeddingDB &operator=(const EmbeddingDB &) = delete;
EmbeddingDB(EmbeddingDB &&) = delete;
EmbeddingDB &operator=(EmbeddingDB &&) = delete;
// Insert a single vector
bool InsertVector(int64_t id, const std::vector<float> &vector, int64_t &allocId);
bool InsertVector(const std::vector<float> &vector, int64_t &allocId); // For auto-increment mode
// Batch insert vectors
std::vector<int64_t> BatchInsertVectors(const std::vector<VectorData> &vectors);
std::vector<int64_t> BatchInsertVectors(const std::vector<std::vector<float>> &vectors); // For auto-increment mode
// Update vector
void UpdateVector(int64_t id, const std::vector<float> &newVector);
// Delete vector
void DeleteVector(int64_t id);
std::vector<FaceSearchResult> SearchSimilarVectors(const std::vector<float> &queryVector, size_t top_k = 3, float keep_similar_threshold = 0.5f,
bool return_feature = false);
// Get vector count
int64_t GetVectorCount() const;
// Get current ID mode
IdMode GetIdMode() const {
return idMode_;
}
bool IsInitialized() const {
return initialized_;
}
// De-initialize database
static void Deinit() {
std::lock_guard<std::mutex> lock(instanceMutex_);
if (instance_) {
instance_.reset();
}
}
std::vector<float> GetVector(int64_t id) const;
void ShowTable();
std::vector<int64_t> GetAllIds();
private:
// Constructor: add ID mode parameter
explicit EmbeddingDB(const std::string &dbPath = ":memory:", size_t vectorDim = 4, const std::string &distanceMetric = "cosine",
IdMode idMode = IdMode::AUTO_INCREMENT);
private:
sqlite3 *db_;
size_t vectorDim_;
std::string tableName_;
IdMode idMode_;
bool initialized_ = false;
// Helper functions
void CheckVectorDimension(const std::vector<float> &vector) const;
void ExecuteSQL(const std::string &sql);
static void CheckSQLiteError(int rc, sqlite3 *db);
int64_t GetLastInsertRowId() const;
private:
// Singleton related
static std::unique_ptr<EmbeddingDB> instance_;
static std::mutex instanceMutex_;
// Database operation mutex
mutable std::mutex dbMutex_;
};
} // namespace inspire
#endif // INSPIRE_EMBEDDING_DB_H

527
cpp-package/inspireface/cpp/inspireface/feature_hub/feature_hub.cpp
View File

@@ -1,527 +0,0 @@
//
// Created by tunm on 2023/9/8.
//
#include "feature_hub.h"
#include "simd.h"
#include "herror.h"
#include <thread>
namespace inspire {
std::mutex FeatureHub::mutex_;
std::shared_ptr<FeatureHub> FeatureHub::instance_ = nullptr;
FeatureHub::FeatureHub() {}
std::shared_ptr<FeatureHub> FeatureHub::GetInstance() {
std::lock_guard<std::mutex> lock(mutex_);
if (!instance_) {
instance_ = std::shared_ptr<FeatureHub>(new FeatureHub());
}
return instance_;
}
int32_t FeatureHub::DisableHub() {
if (!m_enable_) {
INSPIRE_LOGW("FeatureHub is already disabled.");
return HERR_FT_HUB_DISABLE_REPETITION;
}
// Close the database if it starts
if (m_db_) {
int ret = m_db_->CloseDatabase();
if (ret != HSUCCEED) {
INSPIRE_LOGE("Failed to close the database: %d", ret);
return ret;
}
m_db_.reset();
}
m_feature_matrix_list_.clear();
m_search_face_feature_cache_.clear();
m_db_configuration_ = DatabaseConfiguration(); // Reset using the default constructor
m_recognition_threshold_ = 0.0f;
m_search_mode_ = SEARCH_MODE_EAGER;
m_face_feature_ptr_cache_.reset();
m_enable_ = false;
return HSUCCEED;
}
int32_t FeatureHub::EnableHub(const DatabaseConfiguration &configuration, MatrixCore core) {
int32_t ret;
if (m_enable_) {
INSPIRE_LOGW("You have enabled the FeatureHub feature. It is not valid to do so again");
return HERR_FT_HUB_ENABLE_REPETITION;
}
// Config
m_db_configuration_ = configuration;
m_recognition_threshold_ = m_db_configuration_.recognition_threshold;
if (m_recognition_threshold_ < -1.0f || m_recognition_threshold_ > 1.0f) {
INSPIRE_LOGW("The search threshold entered does not fit the required range (-1.0f, 1.0f) and has been set to 0.5 by default");
m_recognition_threshold_ = 0.5f;
}
m_search_mode_ = m_db_configuration_.search_mode;
if (m_db_configuration_.feature_block_num <= 0) {
m_db_configuration_.feature_block_num = 10;
INSPIRE_LOGW(
"The number of feature blocks cannot be 0, but has been set to the default number of 10, that is, the maximum number of stored faces is "
"supported: 5120");
} else if (m_db_configuration_.feature_block_num > 25) {
m_db_configuration_.feature_block_num = 25;
INSPIRE_LOGW(
"The number of feature blocks cannot exceed 25, which has been set to the maximum value, that is, the maximum number of stored faces "
"supported: 12800");
}
// Allocate memory for the feature matrix
for (int i = 0; i < m_db_configuration_.feature_block_num; ++i) {
std::shared_ptr<FeatureBlock> block;
block.reset(FeatureBlock::Create(core, 512, 512));
m_feature_matrix_list_.push_back(block);
}
if (m_db_configuration_.enable_use_db) {
m_db_ = std::make_shared<SQLiteFaceManage>();
if (IsDirectory(m_db_configuration_.db_path)) {
std::string dbFile = m_db_configuration_.db_path + "/" + DB_FILE_NAME;
ret = m_db_->OpenDatabase(dbFile);
} else {
ret = m_db_->OpenDatabase(m_db_configuration_.db_path);
}
if (ret != HSUCCEED) {
INSPIRE_LOGE("An error occurred while opening the database: %d", ret);
return ret;
}
std::vector<FaceFeatureInfo> infos;
ret = m_db_->GetTotalFeatures(infos);
if (ret == HSUCCEED) {
if (!infos.empty()) {
for (auto const &info : infos) {
ret = InsertFaceFeature(info.feature, info.tag, info.customId);
if (ret != HSUCCEED) {
INSPIRE_LOGE("ID: %d, Inserting error: %d", info.customId, ret);
return ret;
}
}
}
m_enable_ = true;
} else {
INSPIRE_LOGE("Failed to get the vector from the database.");
return ret;
}
} else {
m_enable_ = true;
}
m_face_feature_ptr_cache_ = std::make_shared<FaceFeatureEntity>();
return HSUCCEED;
}
int32_t FeatureHub::CosineSimilarity(const std::vector<float> &v1, const std::vector<float> &v2, float &res) {
if (v1.size() != v2.size() || v1.empty()) {
return HERR_SESS_REC_CONTRAST_FEAT_ERR; // The similarity cannot be calculated if the vector lengths are not equal
}
// Calculate the cosine similarity
res = simd_dot(v1.data(), v2.data(), v1.size());
return HSUCCEED;
}
int32_t FeatureHub::CosineSimilarity(const float *v1, const float *v2, int32_t size, float &res) {
res = simd_dot(v1, v2, size);
return HSUCCEED;
}
int32_t FeatureHub::RegisterFaceFeature(const std::vector<float> &feature, int featureIndex, const std::string &tag, int32_t customId) {
if (featureIndex < 0 || featureIndex >= m_feature_matrix_list_.size() * NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_INVALID_INDEX; // Invalid feature index number
}
// Compute which FeatureBlock and which row the feature vector should be stored in
int blockIndex = featureIndex / NUM_OF_FEATURES_IN_BLOCK; // The FeatureBlock where the computation is located
int rowIndex = featureIndex % NUM_OF_FEATURES_IN_BLOCK; // Calculate the line number in the FeatureBlock
// Call the appropriate FeatureBlock registration function
int32_t result = m_feature_matrix_list_[blockIndex]->RegisterFeature(rowIndex, feature, tag, customId);
return result;
}
int32_t FeatureHub::InsertFaceFeature(const std::vector<float> &feature, const std::string &tag, int32_t customId) {
int32_t ret = HSUCCEED;
for (int i = 0; i < m_feature_matrix_list_.size(); ++i) {
auto &block = m_feature_matrix_list_[i];
ret = block->AddFeature(feature, tag, customId);
if (ret != HERR_SESS_REC_BLOCK_FULL) {
break;
}
}
return ret;
}
int32_t FeatureHub::SearchFaceFeature(const std::vector<float> &queryFeature, SearchResult &searchResult, float threshold, bool mostSimilar) {
if (queryFeature.size() != NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // Query feature size does not match expectations
}
bool found = false; // Whether matching features are found
float maxScore = -1.0f; // The maximum score is initialized to a negative number
int maxIndex = -1; // The index corresponding to the maximum score
std::string tag = "None";
int maxCid = -1;
for (int blockIndex = 0; blockIndex < m_feature_matrix_list_.size(); ++blockIndex) {
if (m_feature_matrix_list_[blockIndex]->GetUsedCount() == 0) {
// If the FeatureBlock has no used features, skip to the next block
continue;
}
int startIndex = blockIndex * NUM_OF_FEATURES_IN_BLOCK;
SearchResult tempResult;
// Call the appropriate FeatureBlock search function
int32_t result = m_feature_matrix_list_[blockIndex]->SearchNearest(queryFeature, tempResult);
if (result != HSUCCEED) {
// Error
return result;
}
// If you find a higher score feature
if (tempResult.score > maxScore) {
maxScore = tempResult.score;
maxIndex = startIndex + tempResult.index;
tag = tempResult.tag;
maxCid = tempResult.customId;
if (maxScore >= threshold) {
found = true;
if (!mostSimilar) {
// Use Eager-Mode: When the score is greater than or equal to the threshold, stop searching for the next FeatureBlock
break;
}
}
}
}
if (found) {
searchResult.score = maxScore;
searchResult.index = maxIndex;
searchResult.tag = tag;
searchResult.customId = maxCid;
} else {
searchResult.score = -1.0f;
searchResult.index = -1;
searchResult.tag = "None";
searchResult.customId = -1;
}
return HSUCCEED; // No matching feature found but not an error
}
int32_t FeatureHub::SearchFaceFeatureTopK(const std::vector<float> &queryFeature, std::vector<SearchResult> &searchResultList, size_t maxTopK,
float threshold) {
if (queryFeature.size() != NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_FEAT_SIZE_ERR;
}
std::vector<SearchResult> tempResultList;
searchResultList.clear();
for (int blockIndex = 0; blockIndex < m_feature_matrix_list_.size(); ++blockIndex) {
if (m_feature_matrix_list_[blockIndex]->GetUsedCount() == 0) {
continue;
}
tempResultList.clear();
int32_t result = m_feature_matrix_list_[blockIndex]->SearchTopKNearest(queryFeature, maxTopK, tempResultList);
if (result != HSUCCEED) {
return result;
}
for (const SearchResult &result : tempResultList) {
if (result.score >= threshold) {
searchResultList.push_back(result);
}
}
}
std::sort(searchResultList.begin(), searchResultList.end(), [](const SearchResult &a, const SearchResult &b) { return a.score > b.score; });
if (searchResultList.size() > maxTopK) {
searchResultList.resize(maxTopK);
}
return HSUCCEED;
}
int32_t FeatureHub::DeleteFaceFeature(int featureIndex) {
if (featureIndex < 0 || featureIndex >= m_feature_matrix_list_.size() * NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_INVALID_INDEX; // Invalid feature index number
}
// Calculate which FeatureBlock and which row the feature vector should be removed in
int blockIndex = featureIndex / NUM_OF_FEATURES_IN_BLOCK; // The FeatureBlock where the computation is located
int rowIndex = featureIndex % NUM_OF_FEATURES_IN_BLOCK; // Calculate the line number in the FeatureBlock
// Call the appropriate FeatureBlock delete function
int32_t result = m_feature_matrix_list_[blockIndex]->DeleteFeature(rowIndex);
return result;
}
int32_t FeatureHub::GetFaceFeature(int featureIndex, Embedded &feature) {
if (featureIndex < 0 || featureIndex >= m_feature_matrix_list_.size() * NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_INVALID_INDEX; // Invalid feature index number
}
// Calculate which FeatureBlock and which row the feature vector should be removed in
int blockIndex = featureIndex / NUM_OF_FEATURES_IN_BLOCK; // The FeatureBlock where the computation is located
int rowIndex = featureIndex % NUM_OF_FEATURES_IN_BLOCK; // Calculate the line number in the FeatureBlock
int32_t result = m_feature_matrix_list_[blockIndex]->GetFeature(rowIndex, feature);
return result;
}
int32_t FeatureHub::GetFaceEntity(int featureIndex, Embedded &feature, std::string &tag, FEATURE_STATE &status) {
if (featureIndex < 0 || featureIndex >= m_feature_matrix_list_.size() * NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_INVALID_INDEX; // Invalid feature index number
}
// Calculate which FeatureBlock and which row the feature vector should be removed in
int blockIndex = featureIndex / NUM_OF_FEATURES_IN_BLOCK; // The FeatureBlock where the computation is located
int rowIndex = featureIndex % NUM_OF_FEATURES_IN_BLOCK; // Calculate the line number in the FeatureBlock
int32_t result = m_feature_matrix_list_[blockIndex]->GetFeature(rowIndex, feature);
tag = m_feature_matrix_list_[blockIndex]->GetTagFromRow(rowIndex);
status = m_feature_matrix_list_[blockIndex]->GetStateFromRow(rowIndex);
return result;
}
int32_t FeatureHub::GetFaceFeatureCount() {
int totalFeatureCount = 0;
// Iterate over all FeatureBlocks and add up the number of feature vectors used
for (const auto &block : m_feature_matrix_list_) {
totalFeatureCount += block->GetUsedCount();
}
return totalFeatureCount;
}
int32_t FeatureHub::GetFeatureNum() const {
return NUM_OF_FEATURES_IN_BLOCK;
}
int32_t FeatureHub::UpdateFaceFeature(const std::vector<float> &feature, int featureIndex, const std::string &tag, int32_t customId) {
if (featureIndex < 0 || featureIndex >= m_feature_matrix_list_.size() * NUM_OF_FEATURES_IN_BLOCK) {
return HERR_SESS_REC_INVALID_INDEX; // Invalid feature index number
}
// Calculate which FeatureBlock and which row the feature vector should be removed in
int blockIndex = featureIndex / NUM_OF_FEATURES_IN_BLOCK; // The FeatureBlock where the computation is located
int rowIndex = featureIndex % NUM_OF_FEATURES_IN_BLOCK; // Calculate the line number in the FeatureBlock
// Call the appropriate FeatureBlock registration function
int32_t result = m_feature_matrix_list_[blockIndex]->UpdateFeature(rowIndex, feature, tag, customId);
return result;
}
void FeatureHub::PrintFeatureMatrixInfo() {
m_feature_matrix_list_[0]->PrintMatrix();
}
int32_t FeatureHub::FindFeatureIndexByCustomId(int32_t customId) {
// Iterate over all FeatureBlocks
for (int blockIndex = 0; blockIndex < m_feature_matrix_list_.size(); ++blockIndex) {
int startIndex = blockIndex * NUM_OF_FEATURES_IN_BLOCK;
// Query the customId from the current FeatureBlock
int rowIndex = m_feature_matrix_list_[blockIndex]->FindIndexByCustomId(customId);
if (rowIndex != -1) {
return startIndex + rowIndex; // 返回行号
}
}
return -1; // If none of the featureBlocks is found, -1 is returned
}
int32_t FeatureHub::SearchFaceFeature(const Embedded &queryFeature, SearchResult &searchResult) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
m_search_face_feature_cache_.clear();
std::memset(m_string_cache_, 0, sizeof(m_string_cache_)); // Initial Zero
auto ret = SearchFaceFeature(queryFeature, searchResult, m_recognition_threshold_, m_search_mode_ == SEARCH_MODE_EXHAUSTIVE);
if (ret == HSUCCEED) {
if (searchResult.index != -1) {
ret = GetFaceFeature(searchResult.index, m_search_face_feature_cache_);
}
m_face_feature_ptr_cache_->data = m_search_face_feature_cache_.data();
m_face_feature_ptr_cache_->dataSize = m_search_face_feature_cache_.size();
// Ensure that buffer overflows do not occur
size_t copy_length = std::min(searchResult.tag.size(), sizeof(m_string_cache_) - 1);
std::strncpy(m_string_cache_, searchResult.tag.c_str(), copy_length);
// Make sure the string ends with a null character
m_string_cache_[copy_length] = '\0';
}
return ret;
}
int32_t FeatureHub::SearchFaceFeatureTopK(const Embedded &queryFeature, size_t topK) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
m_top_k_confidence_.clear();
m_top_k_custom_ids_cache_.clear();
auto ret = SearchFaceFeatureTopK(queryFeature, m_search_top_k_cache_, topK, m_recognition_threshold_);
if (ret == HSUCCEED) {
for (int i = 0; i < m_search_top_k_cache_.size(); ++i) {
auto &item = m_search_top_k_cache_[i];
m_top_k_custom_ids_cache_.push_back(item.customId);
m_top_k_confidence_.push_back(item.score);
}
}
return ret;
}
int32_t FeatureHub::FaceFeatureInsertFromCustomId(const std::vector<float> &feature, const std::string &tag, int32_t customId) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto index = FindFeatureIndexByCustomId(customId);
if (index != -1) {
return HERR_SESS_REC_ID_ALREADY_EXIST;
}
auto ret = InsertFaceFeature(feature, tag, customId);
if (ret == HSUCCEED && m_db_ != nullptr) {
// operational database
FaceFeatureInfo item = {0};
item.customId = customId;
item.tag = tag;
item.feature = feature;
ret = m_db_->InsertFeature(item);
}
return ret;
}
int32_t FeatureHub::FaceFeatureRemoveFromCustomId(int32_t customId) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto index = FindFeatureIndexByCustomId(customId);
if (index == -1) {
return HERR_SESS_REC_INVALID_INDEX;
}
auto ret = DeleteFaceFeature(index);
if (ret == HSUCCEED && m_db_ != nullptr) {
ret = m_db_->DeleteFeature(customId);
}
return ret;
}
int32_t FeatureHub::FaceFeatureUpdateFromCustomId(const std::vector<float> &feature, const std::string &tag, int32_t customId) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto index = FindFeatureIndexByCustomId(customId);
if (index == -1) {
return HERR_SESS_REC_INVALID_INDEX;
}
auto ret = UpdateFaceFeature(feature, index, tag, customId);
if (ret == HSUCCEED && m_db_ != nullptr) {
FaceFeatureInfo item = {0};
item.customId = customId;
item.tag = tag;
item.feature = feature;
ret = m_db_->UpdateFeature(item);
}
return ret;
}
int32_t FeatureHub::GetFaceFeatureFromCustomId(int32_t customId) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto index = FindFeatureIndexByCustomId(customId);
if (index == -1) {
return HERR_SESS_REC_INVALID_INDEX;
}
m_getter_face_feature_cache_.clear();
std::string tag;
FEATURE_STATE status;
auto ret = GetFaceEntity(index, m_getter_face_feature_cache_, tag, status);
m_face_feature_ptr_cache_->data = m_getter_face_feature_cache_.data();
m_face_feature_ptr_cache_->dataSize = m_getter_face_feature_cache_.size();
// Ensure that buffer overflows do not occur
size_t copy_length = std::min(tag.size(), sizeof(m_string_cache_) - 1);
std::strncpy(m_string_cache_, tag.c_str(), copy_length);
// Make sure the string ends with a null character
m_string_cache_[copy_length] = '\0';
return ret;
}
int32_t FeatureHub::ViewDBTable() {
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto ret = m_db_->ViewTotal();
return ret;
}
void FeatureHub::SetRecognitionThreshold(float threshold) {
m_recognition_threshold_ = threshold;
}
void FeatureHub::SetRecognitionSearchMode(SearchMode mode) {
m_search_mode_ = mode;
}
// =========== Getter ===========
const Embedded &FeatureHub::GetSearchFaceFeatureCache() const {
return m_search_face_feature_cache_;
}
char *FeatureHub::GetStringCache() {
return m_string_cache_;
}
const std::shared_ptr<FaceFeaturePtr> &FeatureHub::GetFaceFeaturePtrCache() const {
return m_face_feature_ptr_cache_;
}
std::vector<float> &FeatureHub::GetTopKConfidence() {
return m_top_k_confidence_;
}
std::vector<int32_t> &FeatureHub::GetTopKCustomIdsCache() {
return m_top_k_custom_ids_cache_;
}
} // namespace inspire

352
cpp-package/inspireface/cpp/inspireface/feature_hub/feature_hub.h
View File

@@ -1,352 +0,0 @@
//
// Created by tunm on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_FACERECOGNITION_H
#define HYPERFACEREPO_FACERECOGNITION_H
#include <mutex>
#include "common/face_info/face_object.h"
#include "common/face_data/data_tools.h"
#include "middleware/camera_stream/camera_stream.h"
#include "feature_hub/features_block/feature_block.h"
#include "feature_hub/persistence/sqlite_faces_manage.h"
#include "middleware/model_archive/inspire_archive.h"
/**
* @def DB_FILE_NAME
* @brief Default database file name used in the FaceContext.
*/
#define DB_FILE_NAME ".E63520A95DD5B3892C56DA38C3B28E551D8173FD"
#define FEATURE_HUB FeatureHub::GetInstance()
namespace inspire {
// Comparator function object to sort SearchResult by score (descending order)
struct CompareByScore {
bool operator()(const SearchResult& a, const SearchResult& b) const {
return a.score > b.score;
}
};
typedef enum SearchMode {
SEARCH_MODE_EAGER = 0, // Eager mode: Stops when a vector meets the threshold.
SEARCH_MODE_EXHAUSTIVE, // Exhaustive mode: Searches until the best match is found.
} SearchMode;
/**
* @struct DatabaseConfiguration
* @brief Structure to configure database settings for FaceRecognition.
*/
using DatabaseConfiguration = struct DatabaseConfiguration {
int feature_block_num = 20;
bool enable_use_db = false; ///< Whether to enable data persistence.
std::string db_path; ///< Path to the database file.
float recognition_threshold = 0.48f; ///< Face search threshold
SearchMode search_mode = SEARCH_MODE_EAGER; ///< Search mode
};
/**
* @class FeatureHub
* @brief Service for internal feature vector storage.
*
* This class provides methods for face feature extraction, registration, update, search, and more.
*/
class INSPIRE_API FeatureHub {
private:
static std::mutex mutex_; ///< Mutex lock
static std::shared_ptr<FeatureHub> instance_; ///< FeatureHub Instance
const int32_t NUM_OF_FEATURES_IN_BLOCK = 512; ///< Number of features in each feature block.
FeatureHub(const FeatureHub&) = delete;
FeatureHub& operator=(const FeatureHub&) = delete;
public:
/**
* @brief Enables the feature hub with the specified configuration and matrix core.
*
* This function initializes and configures the feature hub based on the provided database
* configuration and the specified matrix processing core. It prepares the hub for operation,
* setting up necessary resources such as database connections and data processing pipelines.
*
* @param configuration The database configuration settings used to configure the hub.
* @param core The matrix core used for processing, defaulting to OpenCV if not specified.
* @return int32_t Returns a status code indicating success (0) or failure (non-zero).
*/
int32_t EnableHub(const DatabaseConfiguration& configuration, MatrixCore core = MC_OPENCV);
/**
* @brief Disables the feature hub, freeing all associated resources.
*
* This function stops all operations within the hub, releases all occupied resources,
* such as database connections and internal data structures. It is used to safely
* shutdown the hub when it is no longer needed or before the application exits, ensuring
* that all resources are properly cleaned up.
*
* @return int32_t Returns a status code indicating success (0) or failure (non-zero).
*/
int32_t DisableHub();
static std::shared_ptr<FeatureHub> GetInstance();
/**
* @brief Searches for a face feature within stored data.
* @param queryFeature Embedded feature to search for.
* @param searchResult SearchResult object to store search results.
* @return int32_t Status code of the search operation.
*/
int32_t SearchFaceFeature(const Embedded& queryFeature, SearchResult &searchResult);
/**
* @brief Search the stored data for the top k facial features that are most similar.
* @param topK Maximum search
* @return int32_t Status code of the search operation.
*/
int32_t SearchFaceFeatureTopK(const Embedded& queryFeature, size_t topK);
/**
* @brief Inserts a face feature with a custom ID.
* @param feature Vector of floats representing the face feature.
* @param tag String tag associated with the feature.
* @param customId Custom ID for the feature.
* @return int32_t Status code of the insertion operation.
*/
int32_t FaceFeatureInsertFromCustomId(const std::vector<float>& feature, const std::string &tag, int32_t customId);
/**
* @brief Removes a face feature by its custom ID.
* @param customId Custom ID of the feature to remove.
* @return int32_t Status code of the removal operation.
*/
int32_t FaceFeatureRemoveFromCustomId(int32_t customId);
/**
* @brief Updates a face feature by its custom ID.
* @param feature Vector of floats representing the new face feature.
* @param tag String tag associated with the feature.
* @param customId Custom ID of the feature to update.
* @return int32_t Status code of the update operation.
*/
int32_t FaceFeatureUpdateFromCustomId(const std::vector<float>& feature, const std::string &tag, int32_t customId);
/**
* @brief Retrieves a face feature by its custom ID.
* @param customId Custom ID of the feature to retrieve.
* @return int32_t Status code of the retrieval operation.
*/
int32_t GetFaceFeatureFromCustomId(int32_t customId);
/**
* @brief Views the database table containing face data.
* @return int32_t Status code of the operation.
*/
int32_t ViewDBTable();
/**
* @brief Sets the recognition threshold for face recognition.
* @param threshold Float value of the new threshold.
*/
void SetRecognitionThreshold(float threshold);
/**
* @brief Sets the search mode for face recognition.
* @param mode Search mode.
*/
void SetRecognitionSearchMode(SearchMode mode);
/**
* @brief Computes the cosine similarity between two feature vectors.
*
* @param v1 First feature vector.
* @param v2 Second feature vector.
* @param res Output parameter to store the cosine similarity result.
* @return int32_t Status code indicating success (0) or failure.
*/
static int32_t CosineSimilarity(const std::vector<float>& v1, const std::vector<float>& v2, float &res);
/**
* @brief Computes the cosine similarity between two feature vectors.
*
* @param v1 Pointer to the first feature vector.
* @param v2 Pointer to the second feature vector.
* @param size Size of the feature vectors.
* @param res Output parameter to store the cosine similarity result.
* @return int32_t Status code indicating success (0) or failure.
*/
static int32_t CosineSimilarity(const float* v1, const float *v2, int32_t size, float &res);
public:
// Getter Function
/**
* @brief Gets the cache used for search operations in face feature data.
* @return A const reference to the Embedded object containing face feature data for search.
*/
const Embedded& GetSearchFaceFeatureCache() const;
/**
* @brief Gets the cache of face feature pointers.
* @return A shared pointer to the cache of face feature pointers.
*/
const std::shared_ptr<FaceFeaturePtr>& GetFaceFeaturePtrCache() const;
/**
* @brief Gets the cache for temporary string storage.
* @return A pointer to the character array used as a string cache.
*/
char* GetStringCache();
/**
* @brief Gets the number of features in the feature block.
*
* @return int32_t Number of features.
*/
int32_t GetFeatureNum() const;
/**
* @brief Retrieves the total number of facial features stored in the feature block.
*
* @return int32_t Total number of facial features.
*/
int32_t GetFaceFeatureCount();
std::vector<float> &GetTopKConfidence();
std::vector<int32_t> &GetTopKCustomIdsCache();
public:
/**
* @brief Constructor for FeatureHub class.
*/
FeatureHub();
/**
* @brief Registers a facial feature in the feature block.
*
* @param feature Vector of floats representing the feature.
* @param featureIndex Index of the feature in the block.
* @param tag String tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t RegisterFaceFeature(const std::vector<float>& feature, int featureIndex, const std::string &tag, int32_t customId);
/**
* @brief Updates a facial feature in the feature block.
*
* @param feature Vector of floats representing the updated feature.
* @param featureIndex Index of the feature in the block.
* @param tag New string tag for the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t UpdateFaceFeature(const std::vector<float>& feature, int featureIndex, const std::string &tag, int32_t customId);
/**
* @brief Searches for the nearest facial feature in the feature block to a given query feature.
*
* @param queryFeature Query feature vector.
* @param searchResult SearchResult structure to store the search results.
* @param threshold Threshold for considering a match.
* @param mostSimilar Whether to find the most similar feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t SearchFaceFeature(const std::vector<float>& queryFeature, SearchResult &searchResult, float threshold, bool mostSimilar=true);
/**
* Search for the top K face features that are most similar to a given query feature.
* @param queryFeature A vector of floats representing the feature to query against.
* @param searchResultList A reference to a vector where the top K search results will be stored.
* @param maxTopK The maximum number of top results to return.
* @param threshold A float representing the minimum similarity score threshold.
* @return int32_t Returns a status code (0 for success, non-zero for any errors).
*/
int32_t SearchFaceFeatureTopK(const std::vector<float>& queryFeature, std::vector<SearchResult> &searchResultList, size_t maxTopK, float threshold);
/**
* @brief Inserts a facial feature into the feature block.
*
* @param feature Vector of floats representing the feature.
* @param tag String tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t InsertFaceFeature(const std::vector<float>& feature, const std::string &tag, int32_t customId);
/**
* @brief Deletes a facial feature from the feature block.
*
* @param featureIndex Index of the feature to delete.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t DeleteFaceFeature(int featureIndex);
/**
* @brief Retrieves a facial feature from the feature block.
*
* @param featureIndex Index of the feature to retrieve.
* @param feature Output parameter to store the retrieved feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t GetFaceFeature(int featureIndex, Embedded &feature);
/**
* @brief Retrieves a facial entity from the feature block.
*
* @param featureIndex Index of the feature to retrieve.
* @param result Output parameter to store the retrieved entity.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t GetFaceEntity(int featureIndex, Embedded &feature, std::string& tag, FEATURE_STATE& status);
/**
* @brief Finds the index of a feature by its custom ID.
*
* @param customId Custom identifier to search for.
* @return int32_t Index of the feature with the given custom ID, or -1 if not found.
*/
int32_t FindFeatureIndexByCustomId(int32_t customId);
/**
* @brief Prints information about the feature matrix.
*/
void PrintFeatureMatrixInfo();
private:
Embedded m_search_face_feature_cache_; ///< Cache for face feature data used in search operations
Embedded m_getter_face_feature_cache_; ///< Cache for face feature data used in search operations
std::shared_ptr<FaceFeaturePtr> m_face_feature_ptr_cache_; ///< Shared pointer to cache of face feature pointers
char m_string_cache_[256]; ///< Cache for temporary string storage
std::vector<SearchResult> m_search_top_k_cache_; ///<
std::vector<float> m_top_k_confidence_;
std::vector<int32_t> m_top_k_custom_ids_cache_;
private:
std::vector<std::shared_ptr<FeatureBlock>> m_feature_matrix_list_; ///< List of feature blocks.
DatabaseConfiguration m_db_configuration_; ///< Configuration settings for the database
float m_recognition_threshold_{0.48f}; ///< Threshold value for face recognition
SearchMode m_search_mode_{SEARCH_MODE_EAGER}; ///< Flag to determine if the search should find the most similar feature
std::shared_ptr<SQLiteFaceManage> m_db_; ///< Shared pointer to the SQLiteFaceManage object
bool m_enable_{false}; ///< Running status
std::mutex m_res_mtx_; ///< Mutex for thread safety.
};
} // namespace inspire
#endif //HYPERFACEREPO_FACERECOGNITION_H

334
cpp-package/inspireface/cpp/inspireface/feature_hub/feature_hub_db.cpp Normal file
View File

@@ -0,0 +1,334 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "feature_hub_db.h"
#include "simd.h"
#include "herror.h"
#include <thread>
#include "middleware/utils.h"
#include "middleware/system.h"
namespace inspire {
std::mutex FeatureHubDB::mutex_;
std::shared_ptr<FeatureHubDB> FeatureHubDB::instance_ = nullptr;
FeatureHubDB::FeatureHubDB() {}
std::shared_ptr<FeatureHubDB> FeatureHubDB::GetInstance() {
std::lock_guard<std::mutex> lock(mutex_);
if (!instance_) {
instance_ = std::shared_ptr<FeatureHubDB>(new FeatureHubDB());
}
return instance_;
}
int32_t FeatureHubDB::DisableHub() {
if (!m_enable_) {
INSPIRE_LOGW("FeatureHub is already disabled.");
return HSUCCEED;
}
// Close the database if it starts
if (EMBEDDING_DB::GetInstance().IsInitialized()) {
EMBEDDING_DB::Deinit();
// if (ret != HSUCCEED) {
// INSPIRE_LOGE("Failed to close the database: %d", ret);
// return ret;
// }
// m_db_.reset();
}
m_search_face_feature_cache_.clear();
m_db_configuration_ = DatabaseConfiguration(); // Reset using the default constructor
m_recognition_threshold_ = 0.0f;
m_search_mode_ = SEARCH_MODE_EAGER;
m_face_feature_ptr_cache_.reset();
m_enable_ = false;
return HSUCCEED;
}
int32_t FeatureHubDB::GetAllIds() {
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
m_all_ids_ = EMBEDDING_DB::GetInstance().GetAllIds();
return HSUCCEED;
}
int32_t FeatureHubDB::EnableHub(const DatabaseConfiguration &configuration) {
int32_t ret;
if (m_enable_) {
INSPIRE_LOGW("You have enabled the FeatureHub feature. It is not valid to do so again");
return HSUCCEED;
}
// Config
m_db_configuration_ = configuration;
m_recognition_threshold_ = m_db_configuration_.recognition_threshold;
if (m_recognition_threshold_ < -1.0f || m_recognition_threshold_ > 1.0f) {
INSPIRE_LOGW("The search threshold entered does not fit the required range (-1.0f, 1.0f) and has been set to 0.5 by default");
m_recognition_threshold_ = 0.5f;
}
std::string dbFile = ":memory:";
if (m_db_configuration_.enable_persistence) {
if (IsDirectory(m_db_configuration_.persistence_db_path)) {
dbFile = os::PathJoin(m_db_configuration_.persistence_db_path, DB_FILE_NAME);
} else {
dbFile = m_db_configuration_.persistence_db_path;
}
}
EMBEDDING_DB::Init(dbFile, 512, IdMode(configuration.primary_key_mode));
m_enable_ = true;
m_face_feature_ptr_cache_ = std::make_shared<FaceFeatureEntity>();
return HSUCCEED;
}
int32_t FeatureHubDB::CosineSimilarity(const std::vector<float> &v1, const std::vector<float> &v2, float &res, bool normalize) {
if (v1.size() != v2.size() || v1.empty()) {
return HERR_SESS_REC_CONTRAST_FEAT_ERR; // The similarity cannot be calculated if the vector lengths are not equal
}
if (normalize) {
std::vector<float> v1_norm = v1;
std::vector<float> v2_norm = v2;
float mse1 = 0.0f;
float mse2 = 0.0f;
for (const auto &one : v1_norm) {
mse1 += one * one;
}
mse1 = sqrt(mse1);
for (float &one : v1_norm) {
one /= mse1;
}
for (const auto &one : v2_norm) {
mse2 += one * one;
}
mse2 = sqrt(mse2);
for (float &one : v2_norm) {
one /= mse2;
}
res = simd_dot(v1_norm.data(), v2_norm.data(), v1_norm.size());
} else {
// Calculate the cosine similarity
res = simd_dot(v1.data(), v2.data(), v1.size());
}
return HSUCCEED;
}
int32_t FeatureHubDB::CosineSimilarity(const float *v1, const float *v2, int32_t size, float &res, bool normalize) {
if (normalize) {
std::vector<float> v1_norm(v1, v1 + size);
std::vector<float> v2_norm(v2, v2 + size);
float mse1 = 0.0f;
float mse2 = 0.0f;
for (const auto &one : v1_norm) {
mse1 += one * one;
}
mse1 = sqrt(mse1);
for (float &one : v1_norm) {
one /= mse1;
}
for (const auto &one : v2_norm) {
mse2 += one * one;
}
mse2 = sqrt(mse2);
for (float &one : v2_norm) {
one /= mse2;
}
res = simd_dot(v1_norm.data(), v2_norm.data(), v1_norm.size());
} else {
res = simd_dot(v1, v2, size);
}
return HSUCCEED;
}
int32_t FeatureHubDB::GetFaceFeatureCount() {
if (!m_enable_) {
INSPIRE_LOGW("FeatureHub is disabled, please enable it before it can be served");
return 0;
}
int totalFeatureCount = 0;
// Iterate over all FeatureBlocks and add up the number of feature vectors used
totalFeatureCount = EMBEDDING_DB::GetInstance().GetVectorCount();
return totalFeatureCount;
}
int32_t FeatureHubDB::SearchFaceFeature(const Embedded &queryFeature, FaceSearchResult &searchResult, bool returnFeature) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HSUCCEED;
}
m_search_face_feature_cache_.clear();
auto results = EMBEDDING_DB::GetInstance().SearchSimilarVectors(queryFeature, 1, m_recognition_threshold_, returnFeature);
searchResult.id = -1;
if (!results.empty()) {
auto &searched = results[0];
searchResult.similarity = searched.similarity;
searchResult.id = searched.id;
if (returnFeature) {
searchResult.feature = searched.feature;
// copy feature to cache
m_search_face_feature_cache_ = searched.feature;
m_face_feature_ptr_cache_->data = m_search_face_feature_cache_.data();
m_face_feature_ptr_cache_->dataSize = m_search_face_feature_cache_.size();
}
}
return HSUCCEED;
}
int32_t FeatureHubDB::SearchFaceFeatureTopKCache(const Embedded &queryFeature, size_t topK) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
m_top_k_confidence_.clear();
m_top_k_custom_ids_cache_.clear();
auto results = EMBEDDING_DB::GetInstance().SearchSimilarVectors(queryFeature, topK, m_recognition_threshold_, false);
for (size_t i = 0; i < results.size(); i++) {
m_top_k_custom_ids_cache_.push_back(results[i].id);
m_top_k_confidence_.push_back(results[i].similarity);
}
return HSUCCEED;
}
int32_t FeatureHubDB::SearchFaceFeatureTopK(const Embedded &queryFeature, std::vector<FaceSearchResult> &searchResult, size_t topK,
bool returnFeature) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGW("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
searchResult = EMBEDDING_DB::GetInstance().SearchSimilarVectors(queryFeature, topK, m_recognition_threshold_, returnFeature);
return HSUCCEED;
}
int32_t FeatureHubDB::FaceFeatureInsert(const std::vector<float> &feature, int32_t id, int64_t &result_id) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
bool ret = EMBEDDING_DB::GetInstance().InsertVector(id, feature, result_id);
if (!ret) {
result_id = -1;
return HERR_FT_HUB_INSERT_FAILURE;
}
return HSUCCEED;
}
int32_t FeatureHubDB::FaceFeatureRemove(int32_t id) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
EMBEDDING_DB::GetInstance().DeleteVector(id);
return HSUCCEED;
}
int32_t FeatureHubDB::FaceFeatureUpdate(const std::vector<float> &feature, int32_t customId) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
try {
EMBEDDING_DB::GetInstance().UpdateVector(customId, feature);
} catch (const std::exception &e) {
INSPIRE_LOGW("Failed to update face feature, id: %d", customId);
return HERR_FT_HUB_NOT_FOUND_FEATURE;
}
return HSUCCEED;
}
int32_t FeatureHubDB::GetFaceFeature(int32_t id) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
auto vec = EMBEDDING_DB::GetInstance().GetVector(id);
if (vec.empty()) {
return HERR_FT_HUB_NOT_FOUND_FEATURE;
}
m_getter_face_feature_cache_ = vec;
m_face_feature_ptr_cache_->data = m_getter_face_feature_cache_.data();
m_face_feature_ptr_cache_->dataSize = m_getter_face_feature_cache_.size();
return HSUCCEED;
}
int32_t FeatureHubDB::GetFaceFeature(int32_t id, std::vector<float> &feature) {
std::lock_guard<std::mutex> lock(mutex_);
if (!m_enable_) {
INSPIRE_LOGW("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
try {
feature = EMBEDDING_DB::GetInstance().GetVector(id);
} catch (const std::exception &e) {
INSPIRE_LOGW("Failed to get face feature, id: %d", id);
return HERR_FT_HUB_NOT_FOUND_FEATURE;
}
return HSUCCEED;
}
int32_t FeatureHubDB::ViewDBTable() {
if (!m_enable_) {
INSPIRE_LOGE("FeatureHub is disabled, please enable it before it can be served");
return HERR_FT_HUB_DISABLE;
}
EMBEDDING_DB::GetInstance().ShowTable();
return HSUCCEED;
}
void FeatureHubDB::SetRecognitionThreshold(float threshold) {
m_recognition_threshold_ = threshold;
}
void FeatureHubDB::SetRecognitionSearchMode(SearchMode mode) {
m_search_mode_ = mode;
}
// =========== Getter ===========
const Embedded &FeatureHubDB::GetSearchFaceFeatureCache() const {
return m_search_face_feature_cache_;
}
const std::shared_ptr<FaceFeaturePtr> &FeatureHubDB::GetFaceFeaturePtrCache() const {
return m_face_feature_ptr_cache_;
}
std::vector<float> &FeatureHubDB::GetTopKConfidence() {
return m_top_k_confidence_;
}
std::vector<int64_t> &FeatureHubDB::GetTopKCustomIdsCache() {
return m_top_k_custom_ids_cache_;
}
std::vector<int64_t> &FeatureHubDB::GetExistingIds() {
return m_all_ids_;
}
} // namespace inspire

276
cpp-package/inspireface/cpp/inspireface/feature_hub/feature_hub_db.h Normal file
View File

@@ -0,0 +1,276 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef INSPIRE_FEATURE_HUB_DB_H
#define INSPIRE_FEATURE_HUB_DB_H
#include <mutex>
#include <vector>
#include <string>
#include <memory>
#include "data_type.h"
#include "feature_hub/embedding_db/embedding_db.h"
#include "log.h"
// Default database file name used in the FaceContext.
#define DB_FILE_NAME ".feature_hub_db_v0"
#define FEATURE_HUB_DB FeatureHubDB::GetInstance()
namespace inspire {
// Comparator function object to sort SearchResult by score (descending order)
struct CompareByScore {
bool operator()(const FaceSearchResult& a, const FaceSearchResult& b) const {
return a.similarity > b.similarity;
}
};
typedef enum SearchMode {
SEARCH_MODE_EAGER = 0, // Eager mode: Stops when a vector meets the threshold.
SEARCH_MODE_EXHAUSTIVE, // Exhaustive mode: Searches until the best match is found.
} SearchMode;
typedef enum PrimaryKeyMode {
AUTO_INCREMENT = 0, // Auto-increment primary key
MANUAL_INPUT, // Manual input primary key
} PrimaryKeyMode;
/**
* @struct DatabaseConfiguration
* @brief Structure to configure database settings for FaceRecognition.
*/
using DatabaseConfiguration = struct DatabaseConfiguration {
PrimaryKeyMode primary_key_mode = PrimaryKeyMode::AUTO_INCREMENT; ///<
bool enable_persistence = false; ///< Whether to enable data persistence.
std::string persistence_db_path; ///< Path to the database file.
float recognition_threshold = 0.48f; ///< Face search threshold
SearchMode search_mode = SEARCH_MODE_EAGER; ///< Search mode (!!Temporarily unavailable!!)
};
/**
* @class FeatureHub
* @brief Service for internal feature vector storage.
*
* This class provides methods for face feature extraction, registration, update, search, and more.
*/
class INSPIRE_API FeatureHubDB {
private:
static std::mutex mutex_; ///< Mutex lock
static std::shared_ptr<FeatureHubDB> instance_; ///< FeatureHub Instance
FeatureHubDB(const FeatureHubDB&) = delete;
FeatureHubDB& operator=(const FeatureHubDB&) = delete;
public:
/**
* @brief Enables the feature hub with the specified configuration and matrix core.
*
* This function initializes and configures the feature hub based on the provided database
* configuration and the specified matrix processing core. It prepares the hub for operation,
* setting up necessary resources such as database connections and data processing pipelines.
*
* @param configuration The database configuration settings used to configure the hub.
* @param core The matrix core used for processing, defaulting to OpenCV if not specified.
* @return int32_t Returns a status code indicating success (0) or failure (non-zero).
*/
int32_t EnableHub(const DatabaseConfiguration& configuration);
/**
* @brief Disables the feature hub, freeing all associated resources.
*
* This function stops all operations within the hub, releases all occupied resources,
* such as database connections and internal data structures. It is used to safely
* shutdown the hub when it is no longer needed or before the application exits, ensuring
* that all resources are properly cleaned up.
*
* @return int32_t Returns a status code indicating success (0) or failure (non-zero).
*/
int32_t DisableHub();
/**
* @brief Get all ids in the database.
* @param ids Output parameter to store the ids.
* @return int32_t Status code of the operation.
*/
int32_t GetAllIds();
static std::shared_ptr<FeatureHubDB> GetInstance();
/**
* @brief Searches for a face feature within stored data.
* @param queryFeature Embedded feature to search for.
* @param searchResult SearchResult object to store search results.
* @return int32_t Status code of the search operation.
*/
int32_t SearchFaceFeature(const Embedded& queryFeature, FaceSearchResult& searchResult, bool returnFeature = true);
/**
* @brief Search the stored data for the top k facial features that are most similar.
* @param topK Maximum search
* @return int32_t Status code of the search operation.
*/
int32_t SearchFaceFeatureTopKCache(const Embedded& queryFeature, size_t topK);
/**
* @brief Search the stored data for the top k facial features that are most similar.
* @param topK Maximum search
* @return int32_t Status code of the search operation.
*/
int32_t SearchFaceFeatureTopK(const Embedded& queryFeature, std::vector<FaceSearchResult>& searchResult, size_t topK, bool returnFeature = false);
/**
* @brief Inserts a face feature with a custom ID.
* @param feature Vector of floats representing the face feature.
* @param tag String tag associated with the feature.
* @param customId Custom ID for the feature.
* @return int32_t Status code of the insertion operation.
*/
int32_t FaceFeatureInsert(const std::vector<float>& feature, int32_t id, int64_t& result_id);
/**
* @brief Removes a face feature by its custom ID.
* @param customId Custom ID of the feature to remove.
* @return int32_t Status code of the removal operation.
*/
int32_t FaceFeatureRemove(int32_t id);
/**
* @brief Updates a face feature by its custom ID.
* @param feature Vector of floats representing the new face feature.
* @param tag String tag associated with the feature.
* @param customId Custom ID of the feature to update.
* @return int32_t Status code of the update operation.
*/
int32_t FaceFeatureUpdate(const std::vector<float>& feature, int32_t customId);
/**
* @brief Retrieves a face feature by its custom ID.
* @param customId Custom ID of the feature to retrieve.
* @return int32_t Status code of the retrieval operation.
*/
int32_t GetFaceFeature(int32_t id);
/**
* @brief Retrieves a face feature by its custom ID.
* @param customId Custom ID of the feature to retrieve.
* @param feature Vector of floats representing the face feature.
* @return int32_t Status code of the retrieval operation.
*/
int32_t GetFaceFeature(int32_t id, std::vector<float>& feature);
/**
* @brief Views the database table containing face data.
* @return int32_t Status code of the operation.
*/
int32_t ViewDBTable();
/**
* @brief Sets the recognition threshold for face recognition.
* @param threshold Float value of the new threshold.
*/
void SetRecognitionThreshold(float threshold);
/**
* @brief Sets the search mode for face recognition.
* @param mode Search mode.
*/
void SetRecognitionSearchMode(SearchMode mode);
/**
* @brief Computes the cosine similarity between two feature vectors.
*
* @param v1 First feature vector.
* @param v2 Second feature vector.
* @param res Output parameter to store the cosine similarity result.
* @return int32_t Status code indicating success (0) or failure.
*/
static int32_t CosineSimilarity(const std::vector<float>& v1, const std::vector<float>& v2, float& res, bool normalize = false);
/**
* @brief Computes the cosine similarity between two feature vectors.
*
* @param v1 Pointer to the first feature vector.
* @param v2 Pointer to the second feature vector.
* @param size Size of the feature vectors.
* @param res Output parameter to store the cosine similarity result.
* @return int32_t Status code indicating success (0) or failure.
*/
static int32_t CosineSimilarity(const float* v1, const float* v2, int32_t size, float& res, bool normalize = true);
public:
// Getter Function
/**
* @brief Gets the cache used for search operations in face feature data.
* @return A const reference to the Embedded object containing face feature data for search.
*/
const Embedded& GetSearchFaceFeatureCache() const;
/**
* @brief Gets the cache of face feature pointers.
* @return A shared pointer to the cache of face feature pointers.
*/
const std::shared_ptr<FaceFeaturePtr>& GetFaceFeaturePtrCache() const;
/**
* @brief Retrieves the total number of facial features stored in the feature block.
*
* @return int32_t Total number of facial features.
*/
int32_t GetFaceFeatureCount();
/**
* @brief Retrieves the confidence scores for the top k facial features.
* @return A reference to the vector of confidence scores.
*/
std::vector<float>& GetTopKConfidence();
/**
* @brief Retrieves the custom IDs for the top k facial features.
* @return A reference to the vector of custom IDs.
*/
std::vector<int64_t>& GetTopKCustomIdsCache();
/**
* @brief Retrieves the existing ids in the database.
* @return A reference to the vector of existing ids.
*/
std::vector<int64_t>& GetExistingIds();
/**
* @brief Constructor for FeatureHub class.
*/
FeatureHubDB();
/**
* @brief Prints information about the feature matrix.
*/
void PrintFeatureMatrixInfo();
private:
Embedded m_search_face_feature_cache_; ///< Cache for face feature data used in search operations
Embedded m_getter_face_feature_cache_; ///< Cache for face feature data used in search operations
std::shared_ptr<FaceFeaturePtr> m_face_feature_ptr_cache_; ///< Shared pointer to cache of face feature pointers
std::vector<FaceSearchResult> m_search_top_k_cache_; ///< Cache for top k search results
std::vector<float> m_top_k_confidence_; ///< Cache for top k confidence scores
std::vector<int64_t> m_top_k_custom_ids_cache_; ///< Cache for top k custom ids
std::vector<int64_t> m_all_ids_; ///< Cache for all ids
private:
DatabaseConfiguration m_db_configuration_; ///< Configuration settings for the database
float m_recognition_threshold_{0.48f}; ///< Threshold value for face recognition
SearchMode m_search_mode_{SEARCH_MODE_EAGER}; ///< Flag to determine if the search should find the most similar feature
bool m_enable_{false}; ///< Running status
std::mutex m_res_mtx_; ///< Mutex for thread safety.
};
} // namespace inspire
#endif // INSPIRE_FEATURE_HUB_DB_H

48
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/feature_block.cpp
View File

@@ -1,48 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#include "feature_block.h"
#include "log.h"
#include "feature_hub/features_block/implement/feature_block_none.h"
#ifdef FEATURE_BLOCK_ENABLE_OPENCV
#include "feature_hub/features_block/implement/feature_block_opencv.h"
#endif
namespace inspire {
FeatureBlock *FeatureBlock::Create(const MatrixCore crop_type, int32_t features_max, int32_t feature_length) {
FeatureBlock* p = nullptr;
switch (crop_type) {
#ifdef FEATURE_BLOCK_ENABLE_OPENCV
case MC_OPENCV:
p = new FeatureBlockOpenCV(features_max, feature_length);
break;
#endif
#ifdef FEATURE_BLOCK_ENABLE_EIGEN
case MC_EIGEN:
LOGD("Not Implement");
break;
#endif
case MC_NONE:
INSPIRE_LOGD("Not Implement");
break;
}
if (p != nullptr) {
p->m_matrix_core_ = crop_type;
p->m_features_max_ = features_max; // Number of facial features
p->m_feature_length_ = feature_length; // Face feature length (default: 512)
p->m_feature_state_.resize(features_max, FEATURE_STATE::IDLE);
p->m_tag_list_.resize(features_max, "None");
p->m_custom_id_list_.resize(features_max, -1);
} else {
INSPIRE_LOGE("Create FeatureBlock error.");
}
return p;
}
} // namespace hyper

290
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/feature_block.h
View File

@@ -1,290 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#pragma once
#ifndef HYPERFACEREPO_FEATUREBLOCK_H
#define HYPERFACEREPO_FEATUREBLOCK_H
#include <mutex>
#include <iostream>
#include <algorithm>
#include "data_type.h"
namespace inspire {
/**
* @enum MatrixCore
* @brief Enumeration for different types of matrix cores used in feature extraction.
*/
typedef enum {
MC_NONE, ///< C/C++ Native matrix core.
MC_OPENCV, ///< OpenCV Mat based matrix core.
MC_EIGEN, ///< Eigen3 Mat based matrix core.
} MatrixCore;
/**
* @enum FEATURE_STATE
* @brief Enumeration for states of feature slots in the feature block.
*/
typedef enum {
IDLE = 0, ///< Slot is idle.
USED, ///< Slot is used.
} FEATURE_STATE;
/**
* @struct SearchResult
* @brief Structure to store the results of a feature search.
*/
typedef struct SearchResult {
float score = -1.0f; ///< Score of the search result.
int32_t index = -1; ///< Index of the result in the feature block.
std::string tag = "None"; ///< Tag associated with the feature.
int32_t customId = -1; ///< Custom identifier for the feature.
} SearchResult;
/**
* @class FeatureBlock
* @brief Class for managing and operating on a block of facial features.
*
* This class provides methods to add, delete, update, and search facial features
* in a feature block, with thread safety using mutexes.
*/
class INSPIRE_API FeatureBlock {
public:
static FeatureBlock* Create(const MatrixCore crop_type, int32_t features_max = 512, int32_t feature_length = 512);
public:
/**
* @brief Destructor for the FeatureBlock class.
*/
virtual ~FeatureBlock() {}
/**
* @brief Adds a feature to the feature block.
* @param feature Vector of floats representing the feature.
* @param tag String tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature addition.
*/
virtual int32_t AddFeature(const std::vector<float>& feature, const std::string &tag, int32_t customId) {
std::lock_guard<std::mutex> lock(m_mtx_); // Use mutex to protect shared data
return UnsafeAddFeature(feature, tag, customId);
}
/**
* @brief Deletes a feature from the feature block.
* @param rowToDelete Index of the feature to be deleted.
* @return int32_t Status of the feature deletion.
*/
virtual int32_t DeleteFeature(int rowToDelete) {
std::lock_guard<std::mutex> lock(m_mtx_);
return UnsafeDeleteFeature(rowToDelete);
}
/**
* @brief Updates a feature in the feature block.
* @param rowToUpdate Index of the feature to be updated.
* @param newFeature New feature vector to replace the old one.
* @param tag New tag for the updated feature.
* @param customId Custom identifier for the updated feature.
* @return int32_t Status of the feature update.
*/
virtual int32_t UpdateFeature(int rowToUpdate, const std::vector<float>& newFeature, const std::string &tag, int32_t customId) {
std::lock_guard<std::mutex> lock(m_mtx_);
return UnsafeUpdateFeature(rowToUpdate, newFeature, tag, customId);
}
/**
* @brief Registers a feature at a specific index in the feature block.
* @param rowToUpdate Index at which to register the new feature.
* @param feature Feature vector to be registered.
* @param tag Tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature registration.
*/
virtual int32_t RegisterFeature(int rowToUpdate, const std::vector<float>& feature, const std::string &tag, int32_t customId) {
std::lock_guard<std::mutex> lock(m_mtx_);
return UnsafeRegisterFeature(rowToUpdate, feature, tag, customId);
}
/**
* @brief Searches for the nearest feature in the block to a given query feature.
* @param queryFeature Query feature vector.
* @param searchResult SearchResult structure to store the search results.
* @return int32_t Status of the search operation.
*/
virtual int32_t SearchNearest(const std::vector<float>& queryFeature, SearchResult &searchResult) = 0;
/**
* @brief Search the first k features in a block that are closest to a given query feature.
* @param topK Maximum number of similarities
* @param searchResults outputs
* */
virtual int32_t SearchTopKNearest(const std::vector<float>& queryFeature, size_t topK, std::vector<SearchResult> &searchResults) = 0;
/**
* @brief Retrieves a feature from the feature block.
* @param row Index of the feature to retrieve.
* @param feature Vector to store the retrieved feature.
* @return int32_t Status of the retrieval operation.
*/
virtual int32_t GetFeature(int row, std::vector<float>& feature) = 0;
/**
* @brief Prints the size of the feature matrix.
*/
virtual void PrintMatrixSize() = 0;
/**
* @brief Prints the entire feature matrix.
*/
virtual void PrintMatrix() = 0;
public:
/**
* @brief Retrieves the tag associated with a feature at a given row index.
* @param row Index of the feature to retrieve the tag for.
* @return std::string Tag associated with the feature at the given row, or an empty string if the row is invalid.
*/
std::string GetTagFromRow(int row) {
std::lock_guard<std::mutex> lock(m_mtx_); // Ensure thread safety
if (row >= 0 && row < m_tag_list_.size() && m_feature_state_[row] == FEATURE_STATE::USED) {
return m_tag_list_[row];
} else {
return ""; // Return an empty string for invalid row or unused slot
}
}
/**
* @brief Retrieves the state of a feature slot at a given row index.
* @param row Index of the feature slot to retrieve the state for.
* @return FEATURE_STATE State of the feature slot at the given row, or IDLE if the row is invalid.
*/
FEATURE_STATE GetStateFromRow(int row) {
std::lock_guard<std::mutex> lock(m_mtx_); // Ensure thread safety
if (row >= 0 && row < m_feature_state_.size()) {
return m_feature_state_[row];
} else {
return FEATURE_STATE::IDLE; // Treat invalid rows as IDLE
}
}
/**
* @brief Finds the index of the first idle (unused) feature slot.
* @return int Index of the first idle slot, or -1 if no idle slot is found.
*/
int FindFirstIdleIndex() const {
for (int i = 0; i < m_feature_state_.size(); ++i) {
if (m_feature_state_[i] == FEATURE_STATE::IDLE) {
return i; // Find the first IDLE index
}
}
return -1; // No IDLE found
}
/**
* @brief Finds the index of the first used feature slot.
* @return int Index of the first used slot, or -1 if no used slot is found.
*/
int FindFirstUsedIndex() const {
for (int i = 0; i < m_feature_state_.size(); ++i) {
if (m_feature_state_[i] == FEATURE_STATE::USED) {
return i; // Find the first USED index
}
}
return -1; // not fond USED
}
/**
* @brief Counts the number of used feature slots.
* @return int Count of used feature slots.
*/
int GetUsedCount() const {
int usedCount = 0;
for (const FEATURE_STATE& state : m_feature_state_) {
if (state == FEATURE_STATE::USED) {
usedCount++;
}
}
return usedCount;
}
/**
* @brief Checks if all feature slots are used.
* @return bool True if all slots are used, false otherwise.
*/
bool IsUsedFull() const {
int usedCount = GetUsedCount();
return usedCount >= m_features_max_;
}
/**
* @brief Finds the index of a feature slot by its custom ID.
* @param customId The custom ID to search for.
* @return size_t Index of the slot with the given custom ID, or -1 if not found.
*/
size_t FindIndexByCustomId(int32_t customId) {
auto it = std::find(m_custom_id_list_.begin(), m_custom_id_list_.end(), customId);
if (it != m_custom_id_list_.end()) {
return std::distance(m_custom_id_list_.begin(), it); // return index
}
return -1;
}
protected:
/**
* @brief Adds a feature to the feature block without thread safety.
* This method should be overridden in derived classes.
* @param feature Vector of floats representing the feature.
* @param tag String tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature addition.
*/
virtual int32_t UnsafeAddFeature(const std::vector<float>& feature, const std::string &tag, int32_t customId) = 0;
/**
* @brief Registers a feature at a specific index in the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToUpdate Index at which to register the new feature.
* @param feature Feature vector to be registered.
* @param tag Tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature registration.
*/
virtual int32_t UnsafeRegisterFeature(int rowToUpdate, const std::vector<float>& feature, const std::string &tag, int32_t customId) = 0;
/**
* @brief Deletes a feature from the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToDelete Index of the feature to be deleted.
* @return int32_t Status of the feature deletion.
*/
virtual int32_t UnsafeDeleteFeature(int rowToDelete) = 0;
/**
* @brief Updates a feature in the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToUpdate Index of the feature to be updated.
* @param newFeature New feature vector to replace the old one.
* @param tag New tag for the updated feature.
* @param customId Custom identifier for the updated feature.
* @return int32_t Status of the feature update.
*/
virtual int32_t UnsafeUpdateFeature(int rowToUpdate, const std::vector<float>& newFeature, const std::string &tag, int32_t customId) = 0;
protected:
MatrixCore m_matrix_core_; ///< Type of matrix core used.
int32_t m_features_max_; ///< Maximum number of features in the block.
int32_t m_feature_length_; ///< Length of each feature vector.
std::mutex m_mtx_; ///< Mutex for thread safety.
std::vector<FEATURE_STATE> m_feature_state_; ///< State of each feature slot.
std::vector<String> m_tag_list_; ///< List of tags associated with each feature.
std::vector<int32_t> m_custom_id_list_; ///< List of custom IDs associated with each feature.
};
} // namespace hyper
#endif //HYPERFACEREPO_FEATUREBLOCK_H

5
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/implement/feature_block_none.cpp
View File

@@ -1,5 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#include "feature_block_none.h"

24
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/implement/feature_block_none.h
View File

@@ -1,24 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#pragma once
#ifndef HYPERFACEREPO_FEATUREBLOCKNONE_H
#define HYPERFACEREPO_FEATUREBLOCKNONE_H
#include "feature_hub/features_block/feature_block.h"
namespace inspire {
class INSPIRE_API FeatureBlockNone {
public:
private:
};
} // namespace hyper
#endif //HYPERFACEREPO_FEATUREBLOCKNONE_H

246
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/implement/feature_block_opencv.cpp
View File

@@ -1,246 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#include "feature_block_opencv.h"
#include "herror.h"
#include "log.h"
namespace inspire {
FeatureBlockOpenCV::FeatureBlockOpenCV(int32_t features_max, int32_t feature_length)
:m_feature_matrix_(features_max, feature_length, CV_32F, cv::Scalar(0.0f)){
}
int32_t FeatureBlockOpenCV::UnsafeAddFeature(const std::vector<float> &feature, const std::string &tag, int32_t customId) {
if (feature.empty()) {
return HERR_SESS_REC_ADD_FEAT_EMPTY; // If the feature is empty, it is not added
}
if (feature.size() != m_feature_length_) {
return HERR_SESS_REC_FEAT_SIZE_ERR;
}
if (IsUsedFull()) {
return HERR_SESS_REC_BLOCK_FULL;
}
cv::Mat newFeatureMat(1, feature.size(), CV_32FC1);
for (int i = 0; i < feature.size(); ++i) {
newFeatureMat.at<float>(0, i) = feature[i];
}
auto idx = FindFirstIdleIndex(); // Find the first free vector position
if (idx == -1) {
return HERR_SESS_REC_BLOCK_FULL;
}
cv::Mat rowToUpdate = m_feature_matrix_.row(idx);
newFeatureMat.copyTo(rowToUpdate);
m_feature_state_[idx] = FEATURE_STATE::USED; // Set feature vector used
m_tag_list_[idx] = tag;
m_custom_id_list_[idx] = customId;
return HSUCCEED;
}
int32_t FeatureBlockOpenCV::UnsafeDeleteFeature(int rowToDelete) {
if (m_feature_matrix_.empty() || rowToDelete < 0 || rowToDelete >= m_feature_matrix_.rows) {
return HERR_SESS_REC_DEL_FAILURE; // Invalid row numbers or matrices are empty and will not be deleted
}
cv::Mat rowToUpdate = m_feature_matrix_.row(rowToDelete);
if (m_feature_state_[rowToDelete] == FEATURE_STATE::IDLE) {
return HERR_SESS_REC_BLOCK_DEL_FAILURE; // Rows are idle and will not be deleted
}
m_feature_state_[rowToDelete] = FEATURE_STATE::IDLE;
m_custom_id_list_[rowToDelete] = -1;
return HSUCCEED;
}
int32_t FeatureBlockOpenCV::UnsafeRegisterFeature(int rowToUpdate, const std::vector<float> &feature, const std::string &tag, int32_t customId) {
if (rowToUpdate < 0 || rowToUpdate >= m_feature_matrix_.rows) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // Invalid line number, not updated
}
if (feature.size() != m_feature_length_) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // The new feature does not match the expected size and will not be updated
}
cv::Mat rowToUpdateMat = m_feature_matrix_.row(rowToUpdate);
// 将新特征拷贝到指定行
for (int i = 0; i < feature.size(); ++i) {
rowToUpdateMat.at<float>(0, i) = feature[i];
}
m_feature_state_[rowToUpdate] = USED;
m_tag_list_[rowToUpdate] = tag;
m_custom_id_list_[rowToUpdate] = customId;
return 0;
}
int32_t FeatureBlockOpenCV::UnsafeUpdateFeature(int rowToUpdate, const std::vector<float> &newFeature, const std::string &tag, int32_t customId) {
if (rowToUpdate < 0 || rowToUpdate >= m_feature_matrix_.rows) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // Invalid line number, not updated
}
if (newFeature.size() != m_feature_length_) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // The new feature does not match the expected size and will not be updated
}
cv::Mat rowToUpdateMat = m_feature_matrix_.row(rowToUpdate);
if (m_feature_state_[rowToUpdate] == FEATURE_STATE::IDLE) {
return HERR_SESS_REC_BLOCK_UPDATE_FAILURE; // Rows are idle and not updated
}
// Copies the new feature to the specified row
for (int i = 0; i < newFeature.size(); ++i) {
rowToUpdateMat.at<float>(0, i) = newFeature[i];
}
m_tag_list_[rowToUpdate] = tag;
m_custom_id_list_[rowToUpdate] = customId;
return HSUCCEED;
}
int32_t FeatureBlockOpenCV::SearchNearest(const std::vector<float>& queryFeature, SearchResult &searchResult) {
std::lock_guard<std::mutex> lock(m_mtx_);
if (queryFeature.size() != m_feature_length_) {
return HERR_SESS_REC_FEAT_SIZE_ERR;
}
if (GetUsedCount() == 0) {
return HSUCCEED;
}
cv::Mat queryMat(queryFeature.size(), 1, CV_32FC1, (void*)queryFeature.data());
// Calculate the cosine similarity matrix
cv::Mat cosineSimilarities;
cv::gemm(m_feature_matrix_, queryMat, 1, cv::Mat(), 0, cosineSimilarities);
// Asserts that cosineSimilarities are the vector of m_features_max_ x 1
assert(cosineSimilarities.rows == m_features_max_ && cosineSimilarities.cols == 1);
// Used to store similarity scores and their indexes
std::vector<std::pair<float, int>> similarityScores;
for (int i = 0; i < m_features_max_; ++i) {
// Check whether the status is IDLE
if (m_feature_state_[i] == FEATURE_STATE::IDLE) {
continue; // Skip the eigenvector of IDLE state
}
// Gets the similarity score for line i
float similarityScore = cosineSimilarities.at<float>(i, 0);
// Adds the similarity score and index to the vector as a pair
similarityScores.push_back(std::make_pair(similarityScore, i));
}
// Find the index of the largest scores in similarityScores
if (!similarityScores.empty()) {
auto maxScoreIter = std::max_element(similarityScores.begin(), similarityScores.end());
float maxScore = maxScoreIter->first;
int maxScoreIndex = maxScoreIter->second;
// Sets the value in the searchResult
searchResult.score = maxScore;
searchResult.index = maxScoreIndex;
searchResult.tag = m_tag_list_[maxScoreIndex];
searchResult.customId = m_custom_id_list_[maxScoreIndex];
return HSUCCEED; // Indicates that the maximum score is found
}
searchResult.score = -1.0f;
searchResult.index = -1;
return HSUCCEED;
}
int32_t FeatureBlockOpenCV::SearchTopKNearest(const std::vector<float> &queryFeature, size_t topK, std::vector<SearchResult> &searchResults) {
std::lock_guard<std::mutex> lock(m_mtx_);
if (queryFeature.size() != m_feature_length_) {
return HERR_SESS_REC_FEAT_SIZE_ERR;
}
if (GetUsedCount() == 0) {
return HSUCCEED;
}
cv::Mat queryMat(queryFeature.size(), 1, CV_32FC1, (void*)queryFeature.data());
// Calculate the cosine similarity matrix
cv::Mat cosineSimilarities;
cv::gemm(m_feature_matrix_, queryMat, 1, cv::Mat(), 0, cosineSimilarities);
// Asserts that cosineSimilarities are the vector of m_features_max_ x 1
assert(cosineSimilarities.rows == m_features_max_ && cosineSimilarities.cols == 1);
// Used to store similarity scores and their indexes
std::vector<std::pair<float, int>> similarityScores;
for (int i = 0; i < m_features_max_; ++i) {
// Check whether the status is IDLE
if (m_feature_state_[i] == FEATURE_STATE::IDLE) {
continue; // Skip the eigenvector of IDLE state
}
// Gets the similarity score for line i
float similarityScore = cosineSimilarities.at<float>(i, 0);
// Adds the similarity score and index to the vector as a pair
similarityScores.push_back(std::make_pair(similarityScore, i));
}
searchResults.clear();
if (similarityScores.size() < topK) {
topK = similarityScores.size();
}
std::partial_sort(similarityScores.begin(), similarityScores.begin() + topK, similarityScores.end(),
[](const std::pair<float, int>& a, const std::pair<float, int>& b) {
return a.first > b.first;
});
for (size_t i = 0; i < topK; i++) {
SearchResult result;
result.score = similarityScores[i].first;
result.index = similarityScores[i].second;
result.tag = m_tag_list_[result.index];
result.customId = m_custom_id_list_[result.index];
searchResults.push_back(result);
}
return HSUCCEED;
}
void FeatureBlockOpenCV::PrintMatrixSize() {
std::cout << m_feature_matrix_.size << std::endl;
}
void FeatureBlockOpenCV::PrintMatrix() {
INSPIRE_LOGD("Num of Features: %d", m_feature_matrix_.cols);
INSPIRE_LOGD("Feature length: %d", m_feature_matrix_.rows);
}
int32_t FeatureBlockOpenCV::GetFeature(int row, std::vector<float> &feature) {
if (row < 0 || row >= m_feature_matrix_.rows) {
return HERR_SESS_REC_FEAT_SIZE_ERR; // Invalid line number, not updated
}
cv::Mat feat = m_feature_matrix_.row(row);
// Copies the new feature to the specified row
for (int i = 0; i < m_feature_length_; ++i) {
feature.push_back(feat.at<float>(0, i));
}
return HSUCCEED;
}
} // namespace hyper

113
cpp-package/inspireface/cpp/inspireface/feature_hub/features_block/implement/feature_block_opencv.h
View File

@@ -1,113 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#pragma once
#ifndef HYPERFACEREPO_FEATUREBLOCKOPENCV_H
#define HYPERFACEREPO_FEATUREBLOCKOPENCV_H
#include "feature_hub/features_block/feature_block.h"
namespace inspire {
/**
* @class FeatureBlockOpenCV
* @brief Class derived from FeatureBlock for managing facial features using OpenCV.
*
* This class provides an implementation of FeatureBlock using OpenCV's Mat data structure
* for storing and manipulating facial features.
*/
class INSPIRE_API FeatureBlockOpenCV : public FeatureBlock{
public:
/**
* @brief Constructor for FeatureBlockOpenCV.
* @param features_max Maximum number of features that can be stored.
* @param feature_length Length of each feature vector.
*/
explicit FeatureBlockOpenCV(int32_t features_max = 512, int32_t feature_length = 512);
/**
* @brief Searches for the nearest feature in the block to a given query feature.
* @param queryFeature Query feature vector.
* @param searchResult SearchResult structure to store the search results.
* @return int32_t Status of the search operation.
*/
int32_t SearchNearest(const std::vector<float>& queryFeature, SearchResult &searchResult) override;
/**
* @brief Search the first k features in a block that are closest to a given query feature.
* @param topK Maximum number of similarities
* @param searchResults outputs
* */
int32_t SearchTopKNearest(const std::vector<float>& queryFeature, size_t topK, std::vector<SearchResult> &searchResults) override;
/**
* @brief Retrieves a feature from the feature block.
* @param row Index of the feature to retrieve.
* @param feature Vector to store the retrieved feature.
* @return int32_t Status of the retrieval operation.
*/
int32_t GetFeature(int row, std::vector<float> &feature) override;
protected:
/**
* @brief Adds a feature to the feature block without thread safety.
* This method should be overridden in derived classes.
* @param feature Vector of floats representing the feature.
* @param tag String tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature addition.
*/
int32_t UnsafeAddFeature(const std::vector<float> &feature, const std::string &tag, int32_t customId) override;
/**
* @brief Registers a feature at a specific index in the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToUpdate Index at which to register the new feature.
* @param feature Feature vector to be registered.
* @param tag Tag associated with the feature.
* @param customId Custom identifier for the feature.
* @return int32_t Status of the feature registration.
*/
int32_t UnsafeDeleteFeature(int rowToDelete) override;
/**
* @brief Deletes a feature from the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToDelete Index of the feature to be deleted.
* @return int32_t Status of the feature deletion.
*/
int32_t UnsafeUpdateFeature(int rowToUpdate, const std::vector<float> &newFeature, const std::string &tag, int32_t customId) override;
/**
* @brief Updates a feature in the feature block without thread safety.
* This method should be overridden in derived classes.
* @param rowToUpdate Index of the feature to be updated.
* @param newFeature New feature vector to replace the old one.
* @param tag New tag for the updated feature.
* @param customId Custom identifier for the updated feature.
* @return int32_t Status of the feature update.
*/
int32_t UnsafeRegisterFeature(int rowToUpdate, const std::vector<float> &feature, const std::string &tag, int32_t customId) override;
public:
/**
* @brief Prints the size of the feature matrix.
*/
void PrintMatrixSize() override;
/**
* @brief Prints the entire feature matrix.
*/
void PrintMatrix() override;
private:
cv::Mat m_feature_matrix_; ///< Matrix for storing feature vectors.
};
} // namespace hyper
#endif //HYPERFACEREPO_FEATUREBLOCKOPENCV_H

346
cpp-package/inspireface/cpp/inspireface/feature_hub/persistence/sqlite_faces_manage.cpp
View File

@@ -1,346 +0,0 @@
//
// Created by Tunm-Air13 on 2023/10/11.
//
#include <iostream>
#include <iomanip> // for std::setw
#include "sqlite_faces_manage.h"
#include "herror.h"
namespace inspire {
SQLiteFaceManage::SQLiteFaceManage() {
}
SQLiteFaceManage::~SQLiteFaceManage() {
CloseDatabase();
// Optionally, you can add logging here if needed:
// LOG_INFO("SQLiteFaceManage object destroyed and database connection closed.");
}
struct SQLiteDeleter {
void operator()(sqlite3* ptr) const {
sqlite3_close(ptr);
}
};
int32_t SQLiteFaceManage::OpenDatabase(const std::string &dbPath) {
sqlite3* rawDb;
if (sqlite3_open(dbPath.c_str(), &rawDb) != SQLITE_OK) {
// Handle error
return HERR_FT_HUB_OPEN_ERROR;
}
m_db_ = std::shared_ptr<sqlite3>(rawDb, SQLiteDeleter());
// Check if the table exists
const char* checkTableSQL = "SELECT name FROM sqlite_master WHERE type='table' AND name='FaceFeatures';";
sqlite3_stmt* stmt = nullptr;
if (sqlite3_prepare_v2(m_db_.get(), checkTableSQL, -1, &stmt, nullptr) != SQLITE_OK) {
INSPIRE_LOGE("Error checking for table existence: %s", sqlite3_errmsg(m_db_.get()));
return HERR_FT_HUB_CHECK_TABLE_ERROR; // Assuming you have this error code
}
int result = sqlite3_step(stmt);
sqlite3_finalize(stmt);
// If table doesn't exist, create it
if (result != SQLITE_ROW) {
return CreateTable();
}
return HSUCCEED;
}
int32_t SQLiteFaceManage::CloseDatabase() {
if (!m_db_) {
// LOGE("Attempted to close an already closed or uninitialized database.");
return HERR_FT_HUB_NOT_OPENED;
}
// Reset the shared_ptr. This will decrease its reference count.
// If this is the last reference, the database will be closed due to the custom deleter.
m_db_.reset();
// Optionally, log that the database was successfully closed
// LOGD("Database successfully closed.");
return HSUCCEED;
}
int32_t SQLiteFaceManage::CreateTable() {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED; // Example error code for unopened database
}
const char* createTableSQL = R"(
CREATE TABLE IF NOT EXISTS FaceFeatures (
customId INTEGER PRIMARY KEY,
tag TEXT,
feature BLOB
)
)";
char* errMsg = nullptr;
int result = sqlite3_exec(m_db_.get(), createTableSQL, 0, 0, &errMsg);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error creating table: %s" , errMsg);
sqlite3_free(errMsg);
return result;
}
// LOGD("Table successfully created or already exists.");
return SQLITE_OK; // or SUCCESS_CODE, based on your error code system
}
int32_t SQLiteFaceManage::InsertFeature(const FaceFeatureInfo& info) {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED; // Example error code for unopened database
}
const char* insertSQL = "INSERT INTO FaceFeatures (customId, tag, feature) VALUES (?, ?, ?)";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), insertSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return result;
}
// Binding values
sqlite3_bind_int(stmt, 1, info.customId);
sqlite3_bind_text(stmt, 2, info.tag.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_blob(stmt, 3, info.feature.data(), info.feature.size() * sizeof(float), SQLITE_STATIC);
result = sqlite3_step(stmt);
if (result != SQLITE_DONE) {
INSPIRE_LOGE("Error inserting new feature: %s" , sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return HERR_FT_HUB_INSERT_FAILURE;
}
// Clean up the statement
sqlite3_finalize(stmt);
// LOGD("Feature successfully inserted.");
return SQLITE_OK; // or SUCCESS_CODE, based on your error code system
}
int32_t SQLiteFaceManage::GetFeature(int32_t customId, FaceFeatureInfo& outInfo) {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED;
}
const char* selectSQL = "SELECT customId, tag, feature FROM FaceFeatures WHERE customId = ?";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), selectSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return HERR_FT_HUB_PREPARING_FAILURE;
}
// Bind the customId to the prepared statement
sqlite3_bind_int(stmt, 1, customId);
result = sqlite3_step(stmt);
if (result == SQLITE_ROW) {
outInfo.customId = sqlite3_column_int(stmt, 0);
outInfo.tag = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
const void* blobData = sqlite3_column_blob(stmt, 2);
int blobSize = sqlite3_column_bytes(stmt, 2) / sizeof(float);
const float* begin = static_cast<const float*>(blobData);
outInfo.feature = std::vector<float>(begin, begin + blobSize);
} else if (result == SQLITE_DONE) {
INSPIRE_LOGE("No feature found with customId: %d", customId);
sqlite3_finalize(stmt);
return HERR_FT_HUB_NO_RECORD_FOUND; // Assuming you have an error code for record not found
} else {
INSPIRE_LOGE("Error executing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return HERR_FT_HUB_EXECUTING_FAILURE;
}
// Clean up the statement
sqlite3_finalize(stmt);
INSPIRE_LOGD("Feature successfully retrieved.");
return HSUCCEED;
}
int32_t SQLiteFaceManage::DeleteFeature(int32_t customId) {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED;
}
const char* deleteSQL = "DELETE FROM FaceFeatures WHERE customId = ?";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), deleteSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return HERR_FT_HUB_PREPARING_FAILURE;
}
// Bind the customId to the prepared statement
sqlite3_bind_int(stmt, 1, customId);
result = sqlite3_step(stmt);
if (result != SQLITE_DONE) {
INSPIRE_LOGE("Error deleting feature with customId: %d, Error: %s", customId, sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return HERR_FT_HUB_EXECUTING_FAILURE;
}
int changes = sqlite3_changes(m_db_.get());
if (changes == 0) {
INSPIRE_LOGE("No feature found with customId: %d. Nothing was deleted.", customId);
sqlite3_finalize(stmt);
return HERR_FT_HUB_NO_RECORD_FOUND; // Assuming you have an error code for record not found
}
// Clean up the statement
sqlite3_finalize(stmt);
// LOGD("Feature with customId: %d successfully deleted.", customId);
return HSUCCEED;
}
int32_t SQLiteFaceManage::UpdateFeature(const FaceFeatureInfo& info) {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED;
}
const char* updateSQL = "UPDATE FaceFeatures SET tag = ?, feature = ? WHERE customId = ?";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), updateSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return HERR_FT_HUB_PREPARING_FAILURE;
}
// Binding values
sqlite3_bind_text(stmt, 1, info.tag.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_blob(stmt, 2, info.feature.data(), info.feature.size() * sizeof(float), SQLITE_STATIC);
sqlite3_bind_int(stmt, 3, info.customId);
result = sqlite3_step(stmt);
if (result != SQLITE_DONE) {
INSPIRE_LOGE("Error updating feature with customId: %d, Error: %s", info.customId, sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return result;
}
int changes = sqlite3_changes(m_db_.get());
if (changes == 0) {
INSPIRE_LOGE("No feature found with customId: %d. Nothing was updated.", info.customId);
sqlite3_finalize(stmt);
return HERR_FT_HUB_NO_RECORD_FOUND; // Assuming you have an error code for record not found
}
// Clean up the statement
sqlite3_finalize(stmt);
// LOGD("Feature with customId: %d successfully updated.", info.customId);
return HSUCCEED;
}
int32_t SQLiteFaceManage::ViewTotal() {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED;
}
const char* selectSQL = "SELECT customId, tag FROM FaceFeatures";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), selectSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return result;
}
// Print table header
std::cout << "+----------+-----------------------+\n";
std::cout << "| customId | tag |\n";
std::cout << "+----------+-----------------------+\n";
while ((result = sqlite3_step(stmt)) == SQLITE_ROW) {
int32_t customId = sqlite3_column_int(stmt, 0);
const unsigned char* tag = sqlite3_column_text(stmt, 1);
std::cout << "| " << std::setw(8) << customId << " | " << std::setw(21) << tag << " |\n";
}
std::cout << "+----------+-----------------------+\n";
if (result != SQLITE_DONE) {
INSPIRE_LOGE("Error executing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return HERR_FT_HUB_PREPARING_FAILURE;
}
// Clean up the statement
sqlite3_finalize(stmt);
INSPIRE_LOGD("Successfully displayed all records.");
return HSUCCEED;
}
int32_t SQLiteFaceManage::GetTotalFeatures(std::vector<FaceFeatureInfo>& infoList) {
if (!m_db_) {
INSPIRE_LOGE("Database is not opened. Please open the database first.");
return HERR_FT_HUB_NOT_OPENED;
}
const char* selectSQL = "SELECT customId, tag, feature FROM FaceFeatures";
sqlite3_stmt* stmt = nullptr;
int result = sqlite3_prepare_v2(m_db_.get(), selectSQL, -1, &stmt, nullptr);
if (result != SQLITE_OK) {
INSPIRE_LOGE("Error preparing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
return HERR_FT_HUB_PREPARING_FAILURE;
}
while ((result = sqlite3_step(stmt)) == SQLITE_ROW) {
FaceFeatureInfo featureInfo;
featureInfo.customId = sqlite3_column_int(stmt, 0);
featureInfo.tag = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
const void* blobData = sqlite3_column_blob(stmt, 2);
int blobSize = sqlite3_column_bytes(stmt, 2) / sizeof(float);
const float* begin = static_cast<const float*>(blobData);
featureInfo.feature = std::vector<float>(begin, begin + blobSize);
infoList.push_back(featureInfo);
}
if (result != SQLITE_DONE) {
INSPIRE_LOGE("Error executing the SQL statement: %s", sqlite3_errmsg(m_db_.get()));
sqlite3_finalize(stmt);
return HERR_FT_HUB_EXECUTING_FAILURE;
}
// Clean up the statement
sqlite3_finalize(stmt);
// LOGD("Successfully retrieved all features.");
return HSUCCEED;
}
} // namespace hyper

124
cpp-package/inspireface/cpp/inspireface/feature_hub/persistence/sqlite_faces_manage.h
View File

@@ -1,124 +0,0 @@
//
// Created by Tunm-Air13 on 2023/10/11.
//
#pragma once
#ifndef HYPERFACEREPO_SQLITEFACEMANAGE_H
#define HYPERFACEREPO_SQLITEFACEMANAGE_H
#include "data_type.h"
#include "log.h"
#include "sqlite3.h" // Include the SQLite3 header
#include <vector>
#include <string>
#include "memory"
namespace inspire {
/**
* @struct FaceFeatureInfo
* @brief Structure to represent information about a facial feature.
*/
typedef struct {
int32_t customId; ///< Custom identifier for the feature.
std::string tag; ///< Tag associated with the feature.
std::vector<float> feature; ///< Vector of floats representing the feature.
} FaceFeatureInfo;
/**
* @class SQLiteFaceManage
* @brief Class for managing facial features using SQLite database.
*
* This class provides methods to open, close, create tables, insert, retrieve, delete, and update
* facial features in an SQLite database. It also allows viewing the total number of features in the database.
*/
class INSPIRE_API SQLiteFaceManage {
public:
/**
* @brief Constructor for SQLiteFaceManage class.
*/
SQLiteFaceManage();
/**
* @brief Destructor for SQLiteFaceManage class.
*/
~SQLiteFaceManage();
/**
* @brief Opens an SQLite database at the specified path.
*
* @param dbPath Path to the SQLite database file.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t OpenDatabase(const std::string& dbPath);
/**
* @brief Closes the currently open SQLite database.
*
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t CloseDatabase();
/**
* @brief Creates an SQLite table for storing facial features if it doesn't exist.
*
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t CreateTable();
/**
* @brief Inserts a facial feature into the SQLite database.
*
* @param info Information about the facial feature to be inserted.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t InsertFeature(const FaceFeatureInfo& info);
/**
* @brief Retrieves information about a facial feature from the SQLite database by custom ID.
*
* @param customId Custom identifier of the facial feature to retrieve.
* @param outInfo Output parameter to store the retrieved feature information.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t GetFeature(int32_t customId, FaceFeatureInfo& outInfo);
/**
* @brief Deletes a facial feature from the SQLite database by custom ID.
*
* @param customId Custom identifier of the facial feature to delete.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t DeleteFeature(int32_t customId);
/**
* @brief Updates a facial feature in the SQLite database.
*
* @param info Updated information about the facial feature.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t UpdateFeature(const FaceFeatureInfo& info);
/**
* @brief Retrieves information about all facial features stored in the SQLite database.
*
* @param infoList Output parameter to store the list of facial feature information.
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t GetTotalFeatures(std::vector<FaceFeatureInfo>& infoList);
/**
* @brief Displays the total number of facial features stored in the SQLite database.
*
* @return int32_t Status code indicating success (0) or failure.
*/
int32_t ViewTotal();
private:
std::shared_ptr<sqlite3> m_db_; ///< Pointer to the SQLite database.
};
} // namespace inspire
#endif //HYPERFACEREPO_SQLITEFACEMANAGE_H

86
cpp-package/inspireface/cpp/inspireface/feature_hub/simd.h
View File

@@ -9,52 +9,58 @@
#elif defined(__ARM_NEON__)
/* GCC-compatible compiler, targeting ARM with NEON */
#include <arm_neon.h>
#pragma message("USE SSE")
#endif
#if defined(__GNUC__) && \
(defined(__x86_64__) || defined(__i386__) || defined(_MSC_VER))
#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__) || defined(_MSC_VER))
inline float simd_dot(const float *x, const float *y, const long &len) {
//#pragma message("USE SSE")
float inner_prod = 0.0f;
__m128 X, Y, Z; // 128-bit values
__m128 acc = _mm_setzero_ps(); // set to (0, 0, 0, 0)
float temp[4];
// #pragma message("USE SSE")
float inner_prod = 0.0f;
__m128 X, Y, Z; // 128-bit values
__m128 acc = _mm_setzero_ps(); // set to (0, 0, 0, 0)
float temp[4];
long i;
for (i = 0; i + 4 < len; i += 4) {
X = _mm_loadu_ps(x + i); // load chunk of 4 floats
Y = _mm_loadu_ps(y + i);
Z = _mm_mul_ps(X, Y);
acc = _mm_add_ps(acc, Z);
}
_mm_storeu_ps(&temp[0], acc); // store acc into an array
inner_prod = temp[0] + temp[1] + temp[2] + temp[3];
long i;
for (i = 0; i + 4 < len; i += 4) {
X = _mm_loadu_ps(x + i); // load chunk of 4 floats
Y = _mm_loadu_ps(y + i);
Z = _mm_mul_ps(X, Y);
acc = _mm_add_ps(acc, Z);
}
_mm_storeu_ps(&temp[0], acc); // store acc into an array
inner_prod = temp[0] + temp[1] + temp[2] + temp[3];
// add the remaining values
for (; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
// add the remaining values
for (; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
}
#elif defined(__ARM_NEON__)
inline float simd_dot(const float *x, const float *y, const long &len) {
// #pragma message("USE NEON")
float inner_prod = 0.0f;
float32x4_t X, Y, Z; // 128-bit values
float32x4_t acc = vdupq_n_f32(0.0f); // set to (0, 0, 0, 0)
long i;
for (i = 0; i + 4 < len; i += 4) {
X = vld1q_f32(x + i); // load chunk of 4 floats
Y = vld1q_f32(y + i);
Z = vmulq_f32(X, Y);
acc = vaddq_f32(acc, Z);
}
inner_prod = vgetq_lane_f32(acc, 0) + vgetq_lane_f32(acc, 1) + vgetq_lane_f32(acc, 2) + vgetq_lane_f32(acc, 3);
for (; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
}
#else
inline float simd_dot(const float *x, const float *y, const long &len) {
//#pragma message("USE NEON")
float inner_prod = 0.0f;
float32x4_t X, Y, Z; // 128-bit values
float32x4_t acc = vdupq_n_f32(0.0f); // set to (0, 0, 0, 0)
long i;
for (i = 0; i + 4 < len; i += 4) {
X = vld1q_f32(x + i); // load chunk of 4 floats
Y = vld1q_f32(y + i);
Z = vmulq_f32(X, Y);
acc = vaddq_f32(acc, Z);
}
inner_prod = vgetq_lane_f32(acc, 0) + vgetq_lane_f32(acc, 1) +
vgetq_lane_f32(acc, 2) + vgetq_lane_f32(acc, 3);
for (; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
// #pragma message("USE SCALAR")
float inner_prod = 0.0f;
for (long i = 0; i < len; ++i) {
inner_prod += x[i] * y[i];
}
return inner_prod;
}
#endif
#endif

121
cpp-package/inspireface/cpp/inspireface/herror.h
View File

@@ -1,70 +1,79 @@
//
// Created by Tunm-Air13 on 2023/9/11.
//
#ifndef HYPERFACEREPO_HERROR_H
#define HYPERFACEREPO_HERROR_H
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef INSPIRE_FACE_HERROR_H
#define INSPIRE_FACE_HERROR_H
// [Anchor-Begin]
#define HSUCCEED (0) // Success
#define HERR_BASIC_BASE 0X0001 // Basic error types
#define HERR_UNKNOWN HERR_BASIC_BASE // Unknown error
#define HERR_INVALID_PARAM (HERR_BASIC_BASE+1) // Invalid parameter
#define HERR_INVALID_IMAGE_STREAM_HANDLE (HERR_BASIC_BASE+24) // Invalid image stream handle
#define HERR_INVALID_CONTEXT_HANDLE (HERR_BASIC_BASE+25) // Invalid context handle
#define HERR_INVALID_FACE_TOKEN (HERR_BASIC_BASE+30) // Invalid face token
#define HERR_INVALID_FACE_FEATURE (HERR_BASIC_BASE+31) // Invalid face feature
#define HERR_INVALID_FACE_LIST (HERR_BASIC_BASE+32) // Invalid face feature list
#define HERR_INVALID_BUFFER_SIZE (HERR_BASIC_BASE+33) // Invalid copy token
#define HERR_INVALID_IMAGE_STREAM_PARAM (HERR_BASIC_BASE+34) // Invalid image param
#define HERR_INVALID_SERIALIZATION_FAILED (HERR_BASIC_BASE+35) // Invalid face serialization failed
#define HERR_INVALID_DETECTION_INPUT (HERR_BASIC_BASE+36) // Failed to modify detector input size
#define HSUCCEED (0) // Success
#define HERR_BASIC_BASE 0X0001 // Basic error types
#define HERR_UNKNOWN HERR_BASIC_BASE // Unknown error
#define HERR_INVALID_PARAM (HERR_BASIC_BASE + 1) // Invalid parameter
#define HERR_INVALID_IMAGE_STREAM_HANDLE (HERR_BASIC_BASE + 24) // Invalid image stream handle
#define HERR_INVALID_CONTEXT_HANDLE (HERR_BASIC_BASE + 25) // Invalid context handle
#define HERR_INVALID_FACE_TOKEN (HERR_BASIC_BASE + 30) // Invalid face token
#define HERR_INVALID_FACE_FEATURE (HERR_BASIC_BASE + 31) // Invalid face feature
#define HERR_INVALID_FACE_LIST (HERR_BASIC_BASE + 32) // Invalid face feature list
#define HERR_INVALID_BUFFER_SIZE (HERR_BASIC_BASE + 33) // Invalid copy token
#define HERR_INVALID_IMAGE_STREAM_PARAM (HERR_BASIC_BASE + 34) // Invalid image param
#define HERR_INVALID_SERIALIZATION_FAILED (HERR_BASIC_BASE + 35) // Invalid face serialization failed
#define HERR_INVALID_DETECTION_INPUT (HERR_BASIC_BASE + 36) // Failed to modify detector input size
#define HERR_INVALID_IMAGE_BITMAP_HANDLE (HERR_BASIC_BASE + 37) // Invalid image bitmap handle
#define HERR_SESS_BASE 0X500 // Session error types
#define HERR_SESS_FUNCTION_UNUSABLE (HERR_SESS_BASE+2) // Function not usable
#define HERR_SESS_TRACKER_FAILURE (HERR_SESS_BASE+3) // Tracker module not initialized
#define HERR_SESS_INVALID_RESOURCE (HERR_SESS_BASE+10) // Invalid static resource
#define HERR_SESS_NUM_OF_MODELS_NOT_MATCH (HERR_SESS_BASE+11) // Number of models does not match
#define HERR_SESS_LANDMARK_NUM_NOT_MATCH (HERR_SESS_BASE+20) // The number of input landmark points does not match
#define HERR_SESS_BASE 0X500 // Session error types
#define HERR_SESS_FUNCTION_UNUSABLE (HERR_SESS_BASE + 2) // Function not usable
#define HERR_SESS_TRACKER_FAILURE (HERR_SESS_BASE + 3) // Tracker module not initialized
#define HERR_SESS_INVALID_RESOURCE (HERR_SESS_BASE + 10) // Invalid static resource
#define HERR_SESS_NUM_OF_MODELS_NOT_MATCH (HERR_SESS_BASE + 11) // Number of models does not match
#define HERR_SESS_LANDMARK_NUM_NOT_MATCH (HERR_SESS_BASE + 20) // The number of input landmark points does not match
#define HERR_SESS_LANDMARK_NOT_ENABLE (HERR_SESS_BASE + 21) // The number of input landmark points does not match
#define HERR_SESS_KEY_POINT_NUM_NOT_MATCH (HERR_SESS_BASE + 22) // The number of input key points does not match
#define HERR_SESS_PIPELINE_FAILURE (HERR_SESS_BASE+8) // Pipeline module not initialized
#define HERR_SESS_PIPELINE_FAILURE (HERR_SESS_BASE + 8) // Pipeline module not initialized
#define HERR_SESS_REC_EXTRACT_FAILURE (HERR_SESS_BASE+15) // Face feature extraction not registered
#define HERR_SESS_REC_DEL_FAILURE (HERR_SESS_BASE+16) // Face feature deletion failed due to out of range index
#define HERR_SESS_REC_UPDATE_FAILURE (HERR_SESS_BASE+17) // Face feature update failed due to out of range index
#define HERR_SESS_REC_ADD_FEAT_EMPTY (HERR_SESS_BASE+18) // Feature vector for registration cannot be empty
#define HERR_SESS_REC_FEAT_SIZE_ERR (HERR_SESS_BASE+19) // Incorrect length of feature vector for registration
#define HERR_SESS_REC_INVALID_INDEX (HERR_SESS_BASE+20) // Invalid index number
#define HERR_SESS_REC_CONTRAST_FEAT_ERR (HERR_SESS_BASE+23) // Incorrect length of feature vector for comparison
#define HERR_SESS_REC_BLOCK_FULL (HERR_SESS_BASE+24) // Feature vector block full
#define HERR_SESS_REC_BLOCK_DEL_FAILURE (HERR_SESS_BASE+25) // Deletion failed
#define HERR_SESS_REC_BLOCK_UPDATE_FAILURE (HERR_SESS_BASE+26) // Update failed
#define HERR_SESS_REC_ID_ALREADY_EXIST (HERR_SESS_BASE+27) // ID already exists
#define HERR_SESS_REC_EXTRACT_FAILURE (HERR_SESS_BASE + 15) // Face feature extraction not registered
#define HERR_SESS_REC_DEL_FAILURE (HERR_SESS_BASE + 16) // Face feature deletion failed due to out of range index
#define HERR_SESS_REC_UPDATE_FAILURE (HERR_SESS_BASE + 17) // Face feature update failed due to out of range index
#define HERR_SESS_REC_ADD_FEAT_EMPTY (HERR_SESS_BASE + 18) // Feature vector for registration cannot be empty
#define HERR_SESS_REC_FEAT_SIZE_ERR (HERR_SESS_BASE + 19) // Incorrect length of feature vector for registration
#define HERR_SESS_REC_INVALID_INDEX (HERR_SESS_BASE + 20) // Invalid index number
#define HERR_SESS_REC_CONTRAST_FEAT_ERR (HERR_SESS_BASE + 23) // Incorrect length of feature vector for comparison
#define HERR_SESS_REC_BLOCK_FULL (HERR_SESS_BASE + 24) // Feature vector block full
#define HERR_SESS_REC_BLOCK_DEL_FAILURE (HERR_SESS_BASE + 25) // Deletion failed
#define HERR_SESS_REC_BLOCK_UPDATE_FAILURE (HERR_SESS_BASE + 26) // Update failed
#define HERR_SESS_REC_ID_ALREADY_EXIST (HERR_SESS_BASE + 27) // ID already exists
#define HERR_SESS_FACE_DATA_ERROR (HERR_SESS_BASE+30) // Face data parsing
#define HERR_SESS_FACE_DATA_ERROR (HERR_SESS_BASE + 30) // Face data parsing
#define HERR_SESS_FACE_REC_OPTION_ERROR (HERR_SESS_BASE+40) // An optional parameter is incorrect
#define HERR_SESS_FACE_REC_OPTION_ERROR (HERR_SESS_BASE + 40) // An optional parameter is incorrect
#define HERR_FT_HUB_DISABLE (HERR_SESS_BASE+49) // FeatureHub is disabled
#define HERR_FT_HUB_OPEN_ERROR (HERR_SESS_BASE+50) // Database open error
#define HERR_FT_HUB_NOT_OPENED (HERR_SESS_BASE+51) // Database not opened
#define HERR_FT_HUB_NO_RECORD_FOUND (HERR_SESS_BASE+52) // No record found
#define HERR_FT_HUB_CHECK_TABLE_ERROR (HERR_SESS_BASE+53) // Data table check error
#define HERR_FT_HUB_INSERT_FAILURE (HERR_SESS_BASE+54) // Data insertion error
#define HERR_FT_HUB_PREPARING_FAILURE (HERR_SESS_BASE+55) // Data preparation error
#define HERR_FT_HUB_EXECUTING_FAILURE (HERR_SESS_BASE+56) // SQL execution error
#define HERR_FT_HUB_NOT_VALID_FOLDER_PATH (HERR_SESS_BASE+57) // Invalid folder path
#define HERR_FT_HUB_ENABLE_REPETITION (HERR_SESS_BASE+58) // Enable db function repeatedly
#define HERR_FT_HUB_DISABLE_REPETITION (HERR_SESS_BASE+59) // Disable db function repeatedly
#define HERR_FT_HUB_DISABLE (HERR_SESS_BASE + 49) // FeatureHub is disabled
#define HERR_FT_HUB_OPEN_ERROR (HERR_SESS_BASE + 50) // Database open error
#define HERR_FT_HUB_NOT_OPENED (HERR_SESS_BASE + 51) // Database not opened
#define HERR_FT_HUB_NO_RECORD_FOUND (HERR_SESS_BASE + 52) // No record found
#define HERR_FT_HUB_CHECK_TABLE_ERROR (HERR_SESS_BASE + 53) // Data table check error
#define HERR_FT_HUB_INSERT_FAILURE (HERR_SESS_BASE + 54) // Data insertion error
#define HERR_FT_HUB_PREPARING_FAILURE (HERR_SESS_BASE + 55) // Data preparation error
#define HERR_FT_HUB_EXECUTING_FAILURE (HERR_SESS_BASE + 56) // SQL execution error
#define HERR_FT_HUB_NOT_VALID_FOLDER_PATH (HERR_SESS_BASE + 57) // Invalid folder path
#define HERR_FT_HUB_ENABLE_REPETITION (HERR_SESS_BASE + 58) // Enable db function repeatedly
#define HERR_FT_HUB_DISABLE_REPETITION (HERR_SESS_BASE + 59) // Disable db function repeatedly
#define HERR_FT_HUB_NOT_FOUND_FEATURE (HERR_SESS_BASE + 60) // Get face feature error
#define HERR_ARCHIVE_LOAD_FAILURE (HERR_SESS_BASE+80) // Archive load failure
#define HERR_ARCHIVE_LOAD_MODEL_FAILURE (HERR_SESS_BASE+81) // Model load failure
#define HERR_ARCHIVE_FILE_FORMAT_ERROR (HERR_SESS_BASE+82) // The archive format is incorrect
#define HERR_ARCHIVE_REPETITION_LOAD (HERR_SESS_BASE+83) // Do not reload the model
#define HERR_ARCHIVE_NOT_LOAD (HERR_SESS_BASE+84) // Model not loaded
#define HERR_ARCHIVE_LOAD_FAILURE (HERR_SESS_BASE + 80) // Archive load failure
#define HERR_ARCHIVE_LOAD_MODEL_FAILURE (HERR_SESS_BASE + 81) // Model load failure
#define HERR_ARCHIVE_FILE_FORMAT_ERROR (HERR_SESS_BASE + 82) // The archive format is incorrect
#define HERR_ARCHIVE_REPETITION_LOAD (HERR_SESS_BASE + 83) // Do not reload the model
#define HERR_ARCHIVE_NOT_LOAD (HERR_SESS_BASE + 84) // Model not loaded
#define HERR_DEVICE_BASE 0X900 // hardware error
#define HERR_DEVICE_CUDA_NOT_SUPPORT (HERR_DEVICE_BASE + 1) // CUDA not supported
#define HERR_DEVICE_CUDA_TENSORRT_NOT_SUPPORT (HERR_DEVICE_BASE + 2) // CUDA TensorRT not supported
#define HERR_DEVICE_CUDA_UNKNOWN_ERROR (HERR_DEVICE_BASE + 20) // CUDA unknown error
#define HERR_DEVICE_CUDA_DISABLE (HERR_DEVICE_BASE + 21) // CUDA support is disabled
// [Anchor-End]
#endif //HYPERFACEREPO_HERROR_H
#endif // INSPIRE_FACE_HERROR_H

19
cpp-package/inspireface/cpp/inspireface/information.h
View File

@@ -1,12 +1,15 @@
//
// Created by tunm on 2024/1/31.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef HYPERFACEREPO_INFORMATION_H
#define HYPERFACEREPO_INFORMATION_H
#ifndef INSPIRE_FACE_INFORMATION_H
#define INSPIRE_FACE_INFORMATION_H
#define INSPIRE_FACE_VERSION_MAJOR_STR "1"
#define INSPIRE_FACE_VERSION_MINOR_STR "1"
#define INSPIRE_FACE_VERSION_PATCH_STR "7"
#define INSPIRE_FACE_VERSION_MINOR_STR "2"
#define INSPIRE_FACE_VERSION_PATCH_STR "0"
#endif //HYPERFACEREPO_INFORMATION_H
#define INSPIRE_FACE_EXTENDED_INFORMATION "InspireFace[Community Edition]@General - Build Time: 2025-03-25"
#endif // INSPIRE_FACE_INFORMATION_H

15
cpp-package/inspireface/cpp/inspireface/information.h.in
View File

@@ -1,12 +1,15 @@
//
// Created by tunm on 2024/1/31.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef HYPERFACEREPO_INFORMATION_H
#define HYPERFACEREPO_INFORMATION_H
#ifndef INSPIRE_FACE_INFORMATION_H
#define INSPIRE_FACE_INFORMATION_H
#define INSPIRE_FACE_VERSION_MAJOR_STR "@INSPIRE_FACE_VERSION_MAJOR_STR@"
#define INSPIRE_FACE_VERSION_MINOR_STR "@INSPIRE_FACE_VERSION_MINOR_STR@"
#define INSPIRE_FACE_VERSION_PATCH_STR "@INSPIRE_FACE_VERSION_PATCH_STR@"
#endif //HYPERFACEREPO_INFORMATION_H
#define INSPIRE_FACE_EXTENDED_INFORMATION "@EXTENDED_INFORMATION@"
#endif // INSPIRE_FACE_INFORMATION_H

42
cpp-package/inspireface/cpp/inspireface/isf_check.h Normal file
View File

@@ -0,0 +1,42 @@
#ifndef INSPIRE_FACE_CHECK_H
#define INSPIRE_FACE_CHECK_H
#include "log.h"
#include "herror.h"
#define INSPIREFACE_RETURN_IF_ERROR(...) \
do { \
const int32_t _status = (__VA_ARGS__); \
if (_status != HSUCCEED) { \
INSPIRE_LOGE("Error code: %d", _status); \
return _status; \
} \
} while (0)
#define INSPIREFACE_LOG_IF(severity, condition) \
if (condition) \
INSPIRE_LOG##severity
#define INSPIREFACE_CHECK(condition) \
do { \
if (!(condition)) { \
INSPIRE_LOGF("Check failed: (%s)", #condition); \
} \
} while (0)
#define INSPIREFACE_CHECK_MSG(condition, message) \
do { \
if (!(condition)) { \
INSPIRE_LOGF("Check failed: (%s) %s", #condition, message); \
} \
} while (0)
#define INSPIREFACE_CHECK_EQ(a, b) INSPIREFACE_CHECK((a) == (b)) << "Expected equality of these values: " << #a << " vs " << #b
#define INSPIREFACE_CHECK_NE(a, b) INSPIREFACE_CHECK((a) != (b)) << "Expected inequality of these values: " << #a << " vs " << #b
#define INSPIREFACE_CHECK_LE(a, b) INSPIREFACE_CHECK((a) <= (b)) << "Expected " << #a << " <= " << #b
#define INSPIREFAFECE_CHECK_LT(a, b) INSPIREFACE_CHECK((a) < (b)) << "Expected " << #a << " < " << #b
#define INSPIREFAFECE_CHECK_GE(a, b) INSPIREFACE_CHECK((a) >= (b)) << "Expected " << #a << " >= " << #b
#define INSPIREFAFECE_CHECK_GT(a, b) INSPIREFACE_CHECK((a) > (b)) << "Expected " << #a << " > " << #b
#endif // INSPIRE_FACE_CHECK_H

9
cpp-package/inspireface/cpp/inspireface/log.cpp
View File

@@ -1,6 +1,7 @@
//
// Created by tunm on 2024/4/8.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "log.h"
namespace inspire {
@@ -9,4 +10,4 @@ namespace inspire {
LogManager* LogManager::instance = nullptr;
std::mutex LogManager::mutex;
} // namespace inspire
} // namespace inspire

34
cpp-package/inspireface/cpp/inspireface/log.h
View File

@@ -1,5 +1,5 @@
#ifndef LOG_MANAGER_H
#define LOG_MANAGER_H
#ifndef INSPIRE_FACE_LOG_H
#define INSPIRE_FACE_LOG_H
#include <mutex>
#include <string>
@@ -27,9 +27,9 @@
// Standard platform log macros
#define INSPIRE_LOGD(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_DEBUG, __FILENAME__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define INSPIRE_LOGI(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_INFO, "", "", -1, __VA_ARGS__)
#define INSPIRE_LOGW(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_WARN, __FILENAME__, "", __LINE__, __VA_ARGS__)
#define INSPIRE_LOGE(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_ERROR, __FILENAME__, "", __LINE__, __VA_ARGS__)
#define INSPIRE_LOGF(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_FATAL, __FILENAME__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define INSPIRE_LOGW(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_WARN, "", "", -1, __VA_ARGS__)
#define INSPIRE_LOGE(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_ERROR, "", "", -1, __VA_ARGS__)
#define INSPIRE_LOGF(...) inspire::LogManager::getInstance()->logStandard(inspire::ISF_LOG_FATAL, "", "", -1, __VA_ARGS__)
#endif
// Macro to set the global log level
@@ -54,7 +54,7 @@ private:
static std::mutex mutex;
// Private constructor
LogManager() : currentLevel(ISF_LOG_DEBUG) {} // Default log level is DEBUG
LogManager() : currentLevel(ISF_LOG_INFO) {} // Default log level is INFO
public:
// Disable copy construction and assignment
@@ -111,6 +111,12 @@ public:
va_start(args, format);
__android_log_vprint(androidLevel, tag, format, args);
va_end(args);
// If the log level is fatal, flush the error stream and abort the program
if (level == ISF_LOG_FATAL) {
std::flush(std::cerr);
abort();
}
}
#else
// Method for standard platform logging
@@ -139,12 +145,14 @@ public:
printf(": ");
}
// Set text color for different log levels
// Set text color for different log levels, but only if not on iOS
#ifndef TARGET_OS_IOS
if (level == ISF_LOG_ERROR || level == ISF_LOG_FATAL) {
printf("\033[1;31m"); // Red color for errors and fatal issues
} else if (level == ISF_LOG_WARN) {
printf("\033[1;33m"); // Yellow color for warnings
}
#endif
// Print the actual log message
va_list args;
@@ -152,12 +160,20 @@ public:
vprintf(format, args);
va_end(args);
// Reset text color if needed
// Reset text color if needed, but only if not on iOS
#ifndef TARGET_OS_IOS
if (level == ISF_LOG_ERROR || level == ISF_LOG_WARN || level == ISF_LOG_FATAL) {
printf("\033[0m"); // Reset color
}
#endif
printf("\n"); // New line after log message
// If the log level is fatal, flush the error stream and abort the program
if (level == ISF_LOG_FATAL) {
std::flush(std::cerr);
abort();
}
}
#endif
@@ -165,4 +181,4 @@ public:
} // namespace inspire
#endif // LOG_MANAGER_H
#endif // INSPIRE_FACE_LOG_H

274
cpp-package/inspireface/cpp/inspireface/middleware/any_net.h
View File

@@ -1,274 +0,0 @@
//
// Created by tunm on 2023/5/6.
//
#pragma once
#ifndef BIGGUYSMAIN_ANYNET_H
#define BIGGUYSMAIN_ANYNET_H
#include <utility>
#include "../data_type.h"
#include "inference_helper/inference_helper.h"
#include "configurable.h"
#include "opencv2/opencv.hpp"
#include "../log.h"
#include "model_archive/inspire_archive.h"
namespace inspire {
using AnyTensorOutputs = std::vector<std::pair<std::string, std::vector<float>>>;
/**
* @class AnyNet
* @brief Generic neural network class for various inference tasks.
*
* This class provides a general interface for different types of neural networks,
* facilitating loading parameters, initializing models, and executing forward passes.
*/
class INSPIRE_API AnyNet {
CONFIGURABLE_SUPPORT
public:
/**
* @brief Constructor for AnyNet.
* @param name Name of the neural network.
*/
explicit AnyNet(std::string name) : m_name_(std::move(name)) {}
~AnyNet() {
m_nn_inference_->Finalize();
}
/**
* @brief Loads parameters and initializes the model for inference.
* @param param Parameters for network configuration.
* @param model Pointer to the model.
* @param type Type of the inference helper (default: kMnn).
* @return int32_t Status of the loading and initialization process.
*/
int32_t
loadData(InspireModel &model, InferenceHelper::HelperType type = InferenceHelper::kMnn, bool dynamic = false) {
m_infer_type_ = type;
// must
pushData<int>(model.Config(), "model_index", 0);
pushData<std::string>(model.Config(), "input_layer", "");
pushData<std::vector<std::string>>(model.Config(), "outputs_layers", {"",});
pushData<std::vector<int>>(model.Config(), "input_size", {320, 320});
pushData<std::vector<float>>(model.Config(), "mean", {127.5f, 127.5f, 127.5f});
pushData<std::vector<float>>(model.Config(), "norm", {0.0078125f, 0.0078125f, 0.0078125f});
// rarely
pushData<int>(model.Config(), "input_channel", 3);
pushData<int>(model.Config(), "input_image_channel", 3);
pushData<bool>(model.Config(), "nchw", true);
pushData<bool>(model.Config(), "swap_color", false);
pushData<int>(model.Config(), "data_type", InputTensorInfo::InputTensorInfo::kDataTypeImage);
pushData<int>(model.Config(), "input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeFp32);
pushData<int>(model.Config(), "output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeFp32);
pushData<int>(model.Config(), "threads", 1);
m_nn_inference_.reset(InferenceHelper::Create(m_infer_type_));
m_nn_inference_->SetNumThreads(getData<int>("threads"));
#if defined(ISF_GLOBAL_INFERENCE_BACKEND_USE_MNN_CUDA) && !defined(ISF_ENABLE_RKNN)
INSPIRE_LOGW("You have forced the global use of MNN_CUDA as the neural network inference backend");
m_nn_inference_->SetSpecialBackend(InferenceHelper::kMnnCuda);
#endif
m_output_tensor_info_list_.clear();
std::vector<std::string> outputs_layers = getData<std::vector<std::string>>("outputs_layers");
int tensor_type = getData<int>("input_tensor_type");
int out_tensor_type = getData<int>("output_tensor_type");
for (auto &name: outputs_layers) {
m_output_tensor_info_list_.push_back(OutputTensorInfo(name, out_tensor_type));
}
auto ret = m_nn_inference_->Initialize(model.buffer, model.bufferSize, m_input_tensor_info_list_,
m_output_tensor_info_list_);
if (ret != InferenceHelper::kRetOk) {
INSPIRE_LOGE("NN Initialize fail");
return ret;
}
m_input_tensor_info_list_.clear();
InputTensorInfo input_tensor_info(getData<std::string>("input_layer"), tensor_type, getData<bool>("nchw"));
std::vector<int> input_size = getData<std::vector<int>>("input_size");
int width = input_size[0];
int height = input_size[1];
m_input_image_size_ = {width, height};
int channel = getData<int>("input_channel");
if (getData<bool>("nchw")) {
input_tensor_info.tensor_dims = {1, channel, m_input_image_size_.height, m_input_image_size_.width};
} else {
input_tensor_info.tensor_dims = {1, m_input_image_size_.height, m_input_image_size_.width, channel};
}
input_tensor_info.data_type = getData<int>("data_type");
int image_channel = getData<int>("input_image_channel");
input_tensor_info.image_info.channel = image_channel;
std::vector<float> mean = getData<std::vector<float>>("mean");
std::vector<float> norm = getData<std::vector<float>>("norm");
input_tensor_info.normalize.mean[0] = mean[0];
input_tensor_info.normalize.mean[1] = mean[1];
input_tensor_info.normalize.mean[2] = mean[2];
input_tensor_info.normalize.norm[0] = norm[0];
input_tensor_info.normalize.norm[1] = norm[1];
input_tensor_info.normalize.norm[2] = norm[2];
input_tensor_info.image_info.width = width;
input_tensor_info.image_info.height = height;
input_tensor_info.image_info.channel = channel;
input_tensor_info.image_info.crop_x = 0;
input_tensor_info.image_info.crop_y = 0;
input_tensor_info.image_info.crop_width = width;
input_tensor_info.image_info.crop_height = height;
input_tensor_info.image_info.is_bgr = getData<bool>("nchw");
input_tensor_info.image_info.swap_color = getData<bool>("swap_color");
m_input_tensor_info_list_.push_back(input_tensor_info);
if (dynamic) {
m_nn_inference_->ResizeInput(m_input_tensor_info_list_);
}
return 0;
}
/**
* @brief Performs a forward pass of the network with given input data.
* @param data The input matrix (image) to process.
* @param outputs Outputs of the network (tensor outputs).
*/
void Forward(const Matrix &data, AnyTensorOutputs &outputs) {
InputTensorInfo &input_tensor_info = getMInputTensorInfoList()[0];
if (m_infer_type_ == InferenceHelper::kRknn) {
// Start by simply implementing a temporary color shift on the outside
if (getData<bool>("swap_color")) {
cv::cvtColor(data, m_cache_, cv::COLOR_BGR2RGB);
input_tensor_info.data = m_cache_.data;
} else {
input_tensor_info.data = data.data;
}
} else {
input_tensor_info.data = data.data;
}
Forward(outputs);
}
/**
* @brief Performs a forward pass of the network.
* @param outputs Outputs of the network (tensor outputs).
*/
void Forward(AnyTensorOutputs &outputs) {
// LOGD("ppPreProcess");
if (m_nn_inference_->PreProcess(m_input_tensor_info_list_) != InferenceHelper::kRetOk) {
INSPIRE_LOGD("PreProcess error");
}
// LOGD("PreProcess");
if (m_nn_inference_->Process(m_output_tensor_info_list_) != InferenceHelper::kRetOk) {
INSPIRE_LOGD("Process error");
}
// LOGD("Process");
for (int i = 0; i < m_output_tensor_info_list_.size(); ++i) {
std::vector<float> output_score_raw_list(m_output_tensor_info_list_[i].GetDataAsFloat(),
m_output_tensor_info_list_[i].GetDataAsFloat() +
m_output_tensor_info_list_[i].GetElementNum());
// LOGE("m_output_tensor_info_list_[i].GetElementNum(): %d",m_output_tensor_info_list_[i].GetElementNum());
outputs.push_back(std::make_pair(m_output_tensor_info_list_[i].name, output_score_raw_list));
}
m_cache_.release();
}
public:
/**
* @brief Gets a reference to the input tensor information list.
* @return Reference to the vector of input tensor information.
*/
std::vector<InputTensorInfo> &getMInputTensorInfoList() {
return m_input_tensor_info_list_;
}
/**
* @brief Gets a reference to the output tensor information list.
* @return Reference to the vector of output tensor information.
*/
std::vector<OutputTensorInfo> &getMOutputTensorInfoList() {
return m_output_tensor_info_list_;
}
/**
* @brief Gets the size of the input image.
* @return Size of the input image.
*/
cv::Size &getMInputImageSize() {
return m_input_image_size_;
}
protected:
std::string m_name_; ///< Name of the neural network.
private:
InferenceHelper::HelperType m_infer_type_; ///< Inference engine type
std::shared_ptr<InferenceHelper> m_nn_inference_; ///< Shared pointer to the inference helper.
std::vector<InputTensorInfo> m_input_tensor_info_list_; ///< List of input tensor information.
std::vector<OutputTensorInfo> m_output_tensor_info_list_; ///< List of output tensor information.
cv::Size m_input_image_size_{}; ///< Size of the input image.
cv::Mat m_cache_; ///< Cached matrix for image data.
};
template<typename ImageT, typename TensorT>
AnyTensorOutputs ForwardService(
std::shared_ptr<AnyNet> net,
const ImageT &input,
std::function<void(const ImageT &, TensorT &)> transform) {
InputTensorInfo &input_tensor_info = net->getMInputTensorInfoList()[0];
TensorT transform_tensor;
transform(input, transform_tensor);
input_tensor_info.data = transform_tensor.data; // input tensor only support cv2::Mat
AnyTensorOutputs outputs;
net->Forward(outputs);
return outputs;
}
/**
* @brief Executes a forward pass through the neural network for a given input, with preprocessing.
* @tparam ImageT Type of the input image.
* @tparam TensorT Type of the transformed tensor.
* @tparam PreprocessCallbackT Type of the preprocessing callback function.
* @param net Shared pointer to the AnyNet neural network object.
* @param input The input image to be processed.
* @param callback Preprocessing callback function to be applied to the input.
* @param transform Transformation function to convert the input image to a tensor.
* @return AnyTensorOutputs Outputs of the network (tensor outputs).
*
* This template function handles the preprocessing of the input image, transformation to tensor,
* and then passes it through the neural network to get the output. The function is generic and
* can work with different types of images and tensors, as specified by the template parameters.
*/
template<typename ImageT, typename TensorT, typename PreprocessCallbackT>
AnyTensorOutputs ForwardService(
std::shared_ptr<AnyNet> net,
const ImageT &input,
PreprocessCallbackT &callback,
std::function<void(const ImageT &, TensorT &, PreprocessCallbackT &)> transform) {
InputTensorInfo &input_tensor_info = net->getMInputTensorInfoList()[0];
TensorT transform_tensor;
transform(input, transform_tensor, callback);
input_tensor_info.data = transform_tensor.data; // input tensor only support cv2::Mat
AnyTensorOutputs outputs;
net->Forward(outputs);
return outputs;
}
} // namespace
#endif //BIGGUYSMAIN_ANYNET_H

312
cpp-package/inspireface/cpp/inspireface/middleware/any_net_adapter.h Normal file
View File

@@ -0,0 +1,312 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef INSPIREFACE_ANYNETADAPTER_H
#define INSPIREFACE_ANYNETADAPTER_H
#include <utility>
#include <inspirecv/inspirecv.h>
#include "data_type.h"
#include "inference_wrapper/inference_wrapper.h"
#include "configurable.h"
#include "log.h"
#include "model_archive/inspire_archive.h"
#include "nexus_processor/image_processor.h"
#include "initialization_module/launch.h"
#include "system.h"
namespace inspire {
using AnyTensorOutputs = std::vector<std::pair<std::string, std::vector<float>>>;
/**
* @class AnyNet
* @brief Generic neural network class for various inference tasks.
*
* This class provides a general interface for different types of neural networks,
* facilitating loading parameters, initializing models, and executing forward passes.
*/
class INSPIRE_API AnyNetAdapter {
CONFIGURABLE_SUPPORT
public:
/**
* @brief Constructor for AnyNet.
* @param name Name of the neural network.
*/
explicit AnyNetAdapter(std::string name) : m_name_(std::move(name)) {
m_processor_ = nexus::ImageProcessor::Create();
}
~AnyNetAdapter() {
m_nn_inference_->Finalize();
}
/**
* @brief Loads parameters and initializes the model for inference.
* @param param Parameters for network configuration.
* @param model Pointer to the model.
* @param type Type of the inference helper (default: INFER_MNN).
* @return int32_t Status of the loading and initialization process.
*/
int32_t loadData(InspireModel &model, InferenceWrapper::EngineType type = InferenceWrapper::INFER_MNN, bool dynamic = false) {
m_infer_type_ = type;
// must
pushData<int>(model.Config(), "model_index", 0);
pushData<std::string>(model.Config(), "input_layer", "");
pushData<std::vector<std::string>>(model.Config(), "outputs_layers",
{
"",
});
pushData<std::vector<int>>(model.Config(), "input_size", {320, 320});
pushData<std::vector<float>>(model.Config(), "mean", {127.5f, 127.5f, 127.5f});
pushData<std::vector<float>>(model.Config(), "norm", {0.0078125f, 0.0078125f, 0.0078125f});
// rarely
pushData<int>(model.Config(), "input_channel", 3);
pushData<int>(model.Config(), "input_image_channel", 3);
pushData<bool>(model.Config(), "nchw", true);
pushData<bool>(model.Config(), "swap_color", false);
pushData<int>(model.Config(), "data_type", InputTensorInfo::InputTensorInfo::DataTypeImage);
pushData<int>(model.Config(), "input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeFp32);
pushData<int>(model.Config(), "output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeFp32);
pushData<int>(model.Config(), "infer_backend", 0);
pushData<int>(model.Config(), "threads", 1);
m_nn_inference_.reset(InferenceWrapper::Create(m_infer_type_));
m_nn_inference_->SetNumThreads(getData<int>("threads"));
if (m_infer_type_ == InferenceWrapper::INFER_TENSORRT) {
m_nn_inference_->SetDevice(INSPIRE_LAUNCH->GetCudaDeviceId());
}
#if defined(ISF_GLOBAL_INFERENCE_BACKEND_USE_MNN_CUDA) && !defined(ISF_ENABLE_RKNN)
INSPIRE_LOGW("You have forced the global use of MNN_CUDA as the neural network inference backend");
m_nn_inference_->SetSpecialBackend(InferenceWrapper::MMM_CUDA);
#endif
#if defined(ISF_ENABLE_APPLE_EXTENSION)
m_nn_inference_->SetSpecialBackend(INSPIRE_LAUNCH->GetGlobalCoreMLInferenceMode());
#endif
m_output_tensor_info_list_.clear();
std::vector<std::string> outputs_layers = getData<std::vector<std::string>>("outputs_layers");
int tensor_type = getData<int>("input_tensor_type");
int out_tensor_type = getData<int>("output_tensor_type");
for (auto &name : outputs_layers) {
m_output_tensor_info_list_.push_back(OutputTensorInfo(name, out_tensor_type));
}
int32_t ret;
if (model.loadFilePath) {
auto extensionPath = INSPIRE_LAUNCH->GetExtensionPath();
if (extensionPath.empty()) {
INSPIRE_LOGE("Extension path is empty");
return InferenceWrapper::WrapperError;
}
std::string filePath = os::PathJoin(extensionPath, model.fullname);
ret = m_nn_inference_->Initialize(filePath, m_input_tensor_info_list_, m_output_tensor_info_list_);
} else {
ret = m_nn_inference_->Initialize(model.buffer, model.bufferSize, m_input_tensor_info_list_, m_output_tensor_info_list_);
}
if (ret != InferenceWrapper::WrapperOk) {
INSPIRE_LOGE("NN Initialize fail");
return ret;
}
if (ret != InferenceWrapper::WrapperOk) {
INSPIRE_LOGE("NN Initialize fail");
return ret;
}
m_input_tensor_info_list_.clear();
InputTensorInfo input_tensor_info(getData<std::string>("input_layer"), tensor_type, getData<bool>("nchw"));
std::vector<int> input_size = getData<std::vector<int>>("input_size");
int width = input_size[0];
int height = input_size[1];
m_input_image_size_ = {width, height};
int channel = getData<int>("input_channel");
if (getData<bool>("nchw")) {
input_tensor_info.tensor_dims = {1, channel, m_input_image_size_.GetHeight(), m_input_image_size_.GetWidth()};
} else {
input_tensor_info.tensor_dims = {1, m_input_image_size_.GetHeight(), m_input_image_size_.GetWidth(), channel};
}
input_tensor_info.data_type = getData<int>("data_type");
int image_channel = getData<int>("input_image_channel");
input_tensor_info.image_info.channel = image_channel;
std::vector<float> mean = getData<std::vector<float>>("mean");
std::vector<float> norm = getData<std::vector<float>>("norm");
input_tensor_info.normalize.mean[0] = mean[0];
input_tensor_info.normalize.mean[1] = mean[1];
input_tensor_info.normalize.mean[2] = mean[2];
input_tensor_info.normalize.norm[0] = norm[0];
input_tensor_info.normalize.norm[1] = norm[1];
input_tensor_info.normalize.norm[2] = norm[2];
input_tensor_info.image_info.width = width;
input_tensor_info.image_info.height = height;
input_tensor_info.image_info.channel = channel;
input_tensor_info.image_info.crop_x = 0;
input_tensor_info.image_info.crop_y = 0;
input_tensor_info.image_info.crop_width = width;
input_tensor_info.image_info.crop_height = height;
input_tensor_info.image_info.is_bgr = getData<bool>("nchw");
input_tensor_info.image_info.swap_color = getData<bool>("swap_color");
m_input_tensor_info_list_.push_back(input_tensor_info);
if (dynamic) {
m_nn_inference_->ResizeInput(m_input_tensor_info_list_);
}
return 0;
}
void Forward(const inspirecv::Image &image, AnyTensorOutputs &outputs) {
InputTensorInfo &input_tensor_info = getMInputTensorInfoList()[0];
if (m_infer_type_ == InferenceWrapper::INFER_RKNN) {
if (getData<bool>("swap_color")) {
m_cache_ = image.SwapRB();
input_tensor_info.data = (uint8_t *)m_cache_.Data();
} else {
input_tensor_info.data = (uint8_t *)image.Data();
}
} else {
input_tensor_info.data = (uint8_t *)image.Data();
}
Forward(outputs);
}
/**
* @brief Performs a forward pass of the network.
* @param outputs Outputs of the network (tensor outputs).
*/
void Forward(AnyTensorOutputs &outputs) {
// LOGD("ppPreProcess");
if (m_nn_inference_->PreProcess(m_input_tensor_info_list_) != InferenceWrapper::WrapperOk) {
INSPIRE_LOGD("PreProcess error");
}
// LOGD("PreProcess");
if (m_nn_inference_->Process(m_output_tensor_info_list_) != InferenceWrapper::WrapperOk) {
INSPIRE_LOGD("Process error");
}
// LOGD("Process");
for (int i = 0; i < m_output_tensor_info_list_.size(); ++i) {
std::vector<float> output_score_raw_list(m_output_tensor_info_list_[i].GetDataAsFloat(),
m_output_tensor_info_list_[i].GetDataAsFloat() + m_output_tensor_info_list_[i].GetElementNum());
// LOGE("m_output_tensor_info_list_[i].GetElementNum(): %d",m_output_tensor_info_list_[i].GetElementNum());
outputs.push_back(std::make_pair(m_output_tensor_info_list_[i].name, output_score_raw_list));
}
}
public:
/**
* @brief Gets a reference to the input tensor information list.
* @return Reference to the vector of input tensor information.
*/
std::vector<InputTensorInfo> &getMInputTensorInfoList() {
return m_input_tensor_info_list_;
}
/**
* @brief Gets a reference to the output tensor information list.
* @return Reference to the vector of output tensor information.
*/
std::vector<OutputTensorInfo> &getMOutputTensorInfoList() {
return m_output_tensor_info_list_;
}
/**
* @brief Gets the size of the input image.
* @return Size of the input image.
*/
inspirecv::Size<int> &getMInputImageSize() {
return m_input_image_size_;
}
/**
* @brief Softmax function.
*
* @param input The input vector.
* @return The softmax result.
*/
static std::vector<float> Softmax(const std::vector<float> &input) {
std::vector<float> result;
float sum = 0.0;
// Calculate the exponentials and the sum of exponentials
for (float x : input) {
float exp_x = std::exp(x);
result.push_back(exp_x);
sum += exp_x;
}
// Normalize by dividing each element by the sum
for (float &value : result) {
value /= sum;
}
return result;
}
protected:
std::string m_name_; ///< Name of the neural network.
std::unique_ptr<nexus::ImageProcessor> m_processor_; ///< Assign a nexus processor to each anynet object
private:
InferenceWrapper::EngineType m_infer_type_; ///< Inference engine type
std::shared_ptr<InferenceWrapper> m_nn_inference_; ///< Shared pointer to the inference helper.
std::vector<InputTensorInfo> m_input_tensor_info_list_; ///< List of input tensor information.
std::vector<OutputTensorInfo> m_output_tensor_info_list_; ///< List of output tensor information.
inspirecv::Size<int> m_input_image_size_{}; ///< Size of the input image.
inspirecv::Image m_cache_; ///< Cached matrix for image data.
};
template <typename ImageT, typename TensorT>
AnyTensorOutputs ForwardService(std::shared_ptr<AnyNetAdapter> net, const ImageT &input, std::function<void(const ImageT &, TensorT &)> transform) {
InputTensorInfo &input_tensor_info = net->getMInputTensorInfoList()[0];
TensorT transform_tensor;
transform(input, transform_tensor);
input_tensor_info.data = transform_tensor.data; // input tensor only support cv2::Mat
AnyTensorOutputs outputs;
net->Forward(outputs);
return outputs;
}
/**
* @brief Executes a forward pass through the neural network for a given input, with preprocessing.
* @tparam ImageT Type of the input image.
* @tparam TensorT Type of the transformed tensor.
* @tparam PreprocessCallbackT Type of the preprocessing callback function.
* @param net Shared pointer to the AnyNet neural network object.
* @param input The input image to be processed.
* @param callback Preprocessing callback function to be applied to the input.
* @param transform Transformation function to convert the input image to a tensor.
* @return AnyTensorOutputs Outputs of the network (tensor outputs).
*
* This template function handles the preprocessing of the input image, transformation to tensor,
* and then passes it through the neural network to get the output. The function is generic and
* can work with different types of images and tensors, as specified by the template parameters.
*/
template <typename ImageT, typename TensorT, typename PreprocessCallbackT>
AnyTensorOutputs ForwardService(std::shared_ptr<AnyNetAdapter> net, const ImageT &input, PreprocessCallbackT &callback,
std::function<void(const ImageT &, TensorT &, PreprocessCallbackT &)> transform) {
InputTensorInfo &input_tensor_info = net->getMInputTensorInfoList()[0];
TensorT transform_tensor;
transform(input, transform_tensor, callback);
input_tensor_info.data = transform_tensor.data; // input tensor only support cv2::Mat
AnyTensorOutputs outputs;
net->Forward(outputs);
return outputs;
}
} // namespace inspire
#endif // INSPIREFACE_ANYNETADAPTER_H

363
cpp-package/inspireface/cpp/inspireface/middleware/camera_stream/camera_stream.h
View File

@@ -1,363 +0,0 @@
#ifndef CAMERA_STREAM_H
#define CAMERA_STREAM_H
#include <memory>
#include "MNN/ImageProcess.hpp"
//#include "basic_types.h"
#include "opencv2/opencv.hpp"
#include "log.h"
//
namespace inspire {
/**
* @brief Enum to represent rotation modes.
*/
enum ROTATION_MODE {
ROTATION_0 = 0,
ROTATION_90 = 1,
ROTATION_180 = 2,
ROTATION_270 = 3
};
/**
* @brief Enum to represent rotation modes.
*/
enum DATA_FORMAT {
NV21 = 0, NV12 = 1, RGBA = 2, RGB = 3, BGR = 4, BGRA = 5
};
/**
* @brief A class to handle camera stream and image processing.
*/
class CameraStream {
public:
CameraStream() {
config_.sourceFormat = MNN::CV::YUV_NV21;
config_.destFormat = MNN::CV::BGR;
// config_.filterType = MNN::CV::BICUBIC;
config_.filterType = MNN::CV::BILINEAR;
// config_.filterType = MNN::CV::NEAREST;
config_.wrap = MNN::CV::ZERO;
rotation_mode_ = ROTATION_0;
preview_size_ = 192;
// preview_size_ = 352;
}
/**
* @brief Set the data buffer, height, and width of the camera stream.
*
* @param data_buffer Pointer to the data buffer.
* @param height Height of the image.
* @param width Width of the image.
*/
void SetDataBuffer(const uint8_t *data_buffer, int height, int width) {
this->buffer_ = data_buffer;
this->height_ = height;
this->width_ = width;
preview_scale_ = preview_size_ / static_cast<float>(std::max(height,width));
}
/**
* @brief Set the preview size.
*
* @param size Preview size.
*/
void SetPreviewSize(const int size)
{
preview_size_ = size;
preview_scale_ = preview_size_ / static_cast<float>(std::max(this->height_,this->width_));
}
/**
* @brief Set the rotation mode.
*
* @param mode Rotation mode (e.g., ROTATION_0, ROTATION_90).
*/
void SetRotationMode(ROTATION_MODE mode) { rotation_mode_ = mode; }
/**
* @brief Set the data format.
*
* @param data_format Data format (e.g., NV21, RGBA).
*/
void SetDataFormat(DATA_FORMAT data_format) {
if (data_format == NV21) {
config_.sourceFormat = MNN::CV::YUV_NV21;
}
if (data_format == NV12) {
config_.sourceFormat = MNN::CV::YUV_NV12;
}
if (data_format == RGBA) {
config_.sourceFormat = MNN::CV::RGBA;
}
if (data_format == RGB) {
config_.sourceFormat = MNN::CV::RGB;
}
if (data_format == BGR) {
config_.sourceFormat = MNN::CV::BGR;
}
if (data_format == BGRA) {
config_.sourceFormat = MNN::CV::BGRA;
}
}
/**
* @brief Get an affine-transformed RGB image.
*
* @param affine_matrix Affine transformation matrix.
* @param width_out Width of the output image.
* @param height_out Height of the output image.
* @return cv::Mat Affine-transformed RGB image.
*/
cv::Mat GetAffineRGBImage(const cv::Mat &affine_matrix, const int width_out,
const int height_out) const {
int sw = width_;
int sh = height_;
int rot_sw = sw;
int rot_sh = sh;
MNN::CV::Matrix tr;
assert(affine_matrix.rows == 2);
assert(affine_matrix.cols == 3);
assert(affine_matrix.type() == CV_64F);
cv::Mat trans_matrix;
affine_matrix.convertTo(trans_matrix, CV_32F);
std::vector<float> tr_cv({1, 0, 0, 0, 1, 0, 0, 0, 1});
memcpy(tr_cv.data(), trans_matrix.data, sizeof(float) * 6);
tr.set9(tr_cv.data());
MNN::CV::Matrix tr_inv;
tr.invert(&tr_inv);
std::shared_ptr<MNN::CV::ImageProcess> process(
MNN::CV::ImageProcess::create(config_));
process->setMatrix(tr_inv);
cv::Mat img_out(height_out, width_out, CV_8UC3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
std::vector<int>{1, height_out, width_out, 3}, img_out.data));
process->convert(buffer_, sw, sh, 0, tensor.get());
// std::cout << std::to_string(1) << std::endl;
return img_out;
}
/**
* @brief Get a preview image with optional rotation.
*
* @param with_rotation True if rotation is applied, false otherwise.
* @return cv::Mat Preview image.
*/
cv::Mat GetPreviewImage(bool with_rotation){
return GetScaledImage(preview_scale_ , with_rotation);
}
/**
* @brief Get the preview scale.
*
* @return float Preview scale.
*/
float GetPreviewScale()
{
return preview_scale_;
}
/**
* @brief Get a scaled image with optional rotation.
*
* @param scale Scaling factor.
* @param with_rotation True if rotation is applied, false otherwise.
* @return cv::Mat Scaled image.
*/
cv::Mat GetScaledImage(const float scale, bool with_rotation) {
int sw = width_;
int sh = height_;
int rot_sw = sw;
int rot_sh = sh;
// MNN::CV::Matrix tr;
std::shared_ptr<MNN::CV::ImageProcess> process(
MNN::CV::ImageProcess::create(config_));
if (rotation_mode_ == ROTATION_270 && with_rotation) {
float srcPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ - 1),
(float)(width_ - 1),
0.0f,
(float)(width_ - 1),
(float)(height_ - 1),
};
float dstPoints[] = {(float)(height_ * scale - 1),
0.0f,
0.0f,
0.0f,
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
(float)(width_ * scale - 1)};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
(MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(width_ * scale);
int scaled_width = static_cast<int>(height_ * scale);
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
process->convert(buffer_, sw, sh, 0, tensor.get());
return img_out;
} else if (rotation_mode_ == ROTATION_90 && with_rotation) {
float srcPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ - 1),
(float)(width_ - 1),
0.0f,
(float)(width_ - 1),
(float)(height_ - 1),
};
float dstPoints[] = {
0.0f,
(float)(width_ * scale - 1),
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
0.0f,
(float)(height_ * scale - 1),
0.0f,
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
(MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(width_ * scale);
int scaled_width = static_cast<int>(height_ * scale);
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
process->convert(buffer_, sw, sh, 0, tensor.get());
return img_out;
} else if (rotation_mode_ == ROTATION_180 && with_rotation) {
float srcPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ - 1),
(float)(width_ - 1),
0.0f,
(float)(width_ - 1),
(float)(height_ - 1),
};
float dstPoints[] = {
(float)(width_ * scale - 1),
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
0.0f,
(float)(height_ * scale - 1),
0.0f,
0.0f,
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
(MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(height_ * scale);
int scaled_width = static_cast<int>(width_ * scale);
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
process->convert(buffer_, sw, sh, 0, tensor.get());
return img_out;
} else {
float srcPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ - 1),
(float)(width_ - 1),
0.0f,
(float)(width_ - 1),
(float)(height_ - 1),
};
float dstPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
(float)(width_ * scale - 1),
(float)(height_ * scale - 1),
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints,
(MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(height_ * scale);
int scaled_width = static_cast<int>(width_ * scale);
cv::Mat img_out(scaled_height, scaled_width, CV_8UC3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(
std::vector<int>{1, scaled_height, scaled_width, 3}, img_out.data));
auto err = process->convert(buffer_, sw, sh, 0, tensor.get());
return img_out;
}
}
/**
* @brief Get the affine transformation matrix.
*
* @return cv::Mat Affine transformation matrix.
*/
cv::Mat GetAffineMatrix() const {
cv::Mat affine_matrix(3, 3, CV_32F);
tr_.get9((float *)affine_matrix.data);
cv::Mat affine = affine_matrix.rowRange(0, 2);
cv::Mat affine_64;
affine.convertTo(affine_64, CV_64F);
assert(affine_64.rows == 2);
assert(affine_64.cols == 3);
assert(affine_64.type() == CV_64F);
return affine_64;
}
/**
* @brief Get the height of the camera stream image.
*
* @return int Height.
*/
int GetHeight() const{
return height_;
}
/**
* @brief Get the width of the camera stream image.
*
* @return int Width.
*/
int GetWidth() const{
return width_;
}
/**
* @brief Get the current rotation mode.
*
* @return ROTATION_MODE Current rotation mode.
*/
ROTATION_MODE getRotationMode() const {
return rotation_mode_;
}
private:
const uint8_t *buffer_; ///< Pointer to the data buffer.
int buffer_size_; ///< Size of the data buffer.
std::vector<float> rotation_matrix; ///< Rotation matrix.
int height_; ///< Height of the camera stream image.
int width_; ///< Width of the camera stream image.
float preview_scale_; ///< Scaling factor for the preview image.
int preview_size_; ///< Size of the preview image.
MNN::CV::Matrix tr_; ///< Affine transformation matrix.
ROTATION_MODE rotation_mode_; ///< Current rotation mode.
MNN::CV::ImageProcess::Config config_; ///< Configuration for image processing.
std::shared_ptr<MNN::CV::ImageProcess> process_; ///< Image processing instance.
};
} // inspire
#endif // CAMERA_STREAM_H

18
cpp-package/inspireface/cpp/inspireface/middleware/configurable.cpp
View File

@@ -1,34 +1,32 @@
//
// Created by tunm on 2023/5/5.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "configurable.h"
namespace inspire {
std::string Configurable::toString(int indent) const {
if (indent != 0)
return m_configuration.dump(indent);
else
return m_configuration.dump();
}
std::vector<std::string> Configurable::getNameList() const {
std::vector<std::string> keys;
for (const auto& element : m_configuration.items()) {
for (const auto &element : m_configuration.items()) {
keys.push_back(element.key());
}
return keys;
}
Configurable &Configurable::operator=(const Configurable &other) {
if (this != &other) { // Check the self-assignment
m_configuration = other.m_configuration; // Deep copy using the assignment operator of nlohmann::json
if (this != &other) { // Check the self-assignment
m_configuration = other.m_configuration; // Deep copy using the assignment operator of nlohmann::json
}
return *this;
}
} // namespace hyper
} // namespace inspire

94
cpp-package/inspireface/cpp/inspireface/middleware/configurable.h
View File

@@ -1,10 +1,11 @@
//
// Created by tunm on 2023/5/5.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef HYPERAI_PARAMETER_H
#define HYPERAI_PARAMETER_H
#ifndef INSPIREFACE_CONFIGURABLE_H
#define INSPIREFACE_CONFIGURABLE_H
#include "nlohmann/json.hpp"
#include <iostream>
@@ -24,7 +25,8 @@ namespace inspire {
*/
class INSPIRE_API Configurable {
public:
Configurable() = default;;
Configurable() = default;
;
Configurable(const Configurable& p) : m_configuration(p.m_configuration) {}
@@ -100,7 +102,7 @@ public:
set<std::vector<int>>(key, value.get<std::vector<int>>());
} else if (value[0].is_number_float()) {
set<std::vector<float>>(key, value.get<std::vector<float>>());
} // add more types as needed
} // add more types as needed
// ...
}
}
@@ -108,53 +110,51 @@ public:
}
}
private:
json m_configuration; ///< JSON object to store parameters.
json m_configuration; ///< JSON object to store parameters.
};
#define CONFIGURABLE_SUPPORT \
protected: \
inspire::Configurable m_configuration; \
\
public: \
const inspire::Configurable& getConfiguration() const { \
#define CONFIGURABLE_SUPPORT \
protected: \
inspire::Configurable m_configuration; \
\
public: \
const inspire::Configurable& getConfiguration() const { \
return m_configuration; \
} \
\
void setConfiguration(const inspire::Configurable& param) { \
} \
\
void setConfiguration(const inspire::Configurable& param) { \
m_configuration = param; \
} \
\
bool hasData(const std::string& name) const noexcept { \
return m_configuration.has(name); \
} \
\
template <typename ValueType> \
void setData(const std::string& name, const ValueType& value) { \
} \
\
bool hasData(const std::string& name) const noexcept { \
return m_configuration.has(name); \
} \
\
template <typename ValueType> \
void setData(const std::string& name, const ValueType& value) { \
m_configuration.set<ValueType>(name, value); \
} \
\
template <typename ValueType> \
ValueType getData(const std::string& name) const { \
} \
\
template <typename ValueType> \
ValueType getData(const std::string& name) const { \
return m_configuration.get<ValueType>(name); \
} \
\
template <typename ValueType> \
void pushData(const inspire::Configurable& param, const std::string& name, \
const ValueType& default_value) { \
if (param.has(name)) { \
setData<ValueType>(name, param.get<ValueType>(name)); \
} else { \
setData<ValueType>(name, default_value); \
} \
} \
void loadData(const nlohmann::json& j) { \
} \
\
template <typename ValueType> \
void pushData(const inspire::Configurable& param, const std::string& name, const ValueType& default_value) { \
if (param.has(name)) { \
setData<ValueType>(name, param.get<ValueType>(name)); \
} else { \
setData<ValueType>(name, default_value); \
} \
} \
void loadData(const nlohmann::json& j) { \
m_configuration.load(j); \
} \
std::string toStr(int indent = 4) { \
return m_configuration.toString(indent); \
} \
std::string toStr(int indent = 4) { \
return m_configuration.toString(indent); \
}
} // namespace hyper
#endif //HYPERAI_PARAMETER_H
} // namespace inspire
#endif // INSPIREFACE_CONFIGURABLE_H

36
cpp-package/inspireface/cpp/inspireface/middleware/costman.h
View File

@@ -1,37 +1,37 @@
//
// Created by Tunm-Air13 on 2023/9/21.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef HYPERFACEREPO_TIMER_H
#define HYPERFACEREPO_TIMER_H
#include <opencv2/opencv.hpp>
#ifndef INSPIRE_FACE_COSTMAN_H
#define INSPIRE_FACE_COSTMAN_H
#include <chrono>
namespace inspire {
class Timer {
public:
Timer() {
current_time = (double) cv::getTickCount();
current_time = std::chrono::high_resolution_clock::now();
}
double GetCostTime() const {
return ((double) cv::getTickCount() - current_time) / cv::getTickFrequency() * 1000;
auto now = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now - current_time);
return duration.count() / 1000000.0; // Convert to milliseconds
}
double GetCostTimeUpdate() {
auto cost = ((double) cv::getTickCount() - current_time) / cv::getTickFrequency() * 1000;
current_time = (double) cv::getTickCount();
return cost;
auto now = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now - current_time);
current_time = now;
return duration.count() / 1000000.0; // Convert to milliseconds
}
private:
double current_time;
std::chrono::time_point<std::chrono::high_resolution_clock> current_time;
};
}
} // namespace inspire
#endif //HYPERFACEREPO_TIMER_H
#endif // INSPIRE_FACE_COSTMAN_H

113
cpp-package/inspireface/cpp/inspireface/middleware/cuda_toolkit.h Normal file
View File

@@ -0,0 +1,113 @@
#ifdef ISF_ENABLE_TENSORRT
#ifndef INSPIRE_CUDA_TOOLKIT_H
#define INSPIRE_CUDA_TOOLKIT_H
#include <cuda_runtime_api.h>
#include <NvInfer.h>
#include <log.h>
#include "herror.h"
namespace inspire {
inline static int32_t GetCudaDeviceCount(int32_t *device_count) {
cudaError_t error = cudaGetDeviceCount(device_count);
if (error != cudaSuccess) {
INSPIRE_LOGE("CUDA error: %s", cudaGetErrorString(error));
return HERR_DEVICE_CUDA_UNKNOWN_ERROR;
}
return HSUCCEED;
}
inline static int32_t CheckCudaUsability(int32_t *is_support) {
int device_count;
auto ret = GetCudaDeviceCount(&device_count);
if (ret != HSUCCEED) {
return ret;
}
if (device_count == 0) {
*is_support = 0;
INSPIRE_LOGE("No CUDA devices found");
return HERR_DEVICE_CUDA_NOT_SUPPORT;
}
*is_support = device_count > 0;
return HSUCCEED;
}
inline static int32_t _PrintCudaDeviceInfo() {
try {
INSPIRE_LOGW("TensorRT version: %d.%d.%d", NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH);
// check if CUDA is available
int device_count;
cudaError_t error = cudaGetDeviceCount(&device_count);
if (error != cudaSuccess) {
INSPIRE_LOGE("CUDA error: %s", cudaGetErrorString(error));
return HERR_DEVICE_CUDA_UNKNOWN_ERROR;
}
INSPIRE_LOGW("available CUDA devices: %d", device_count);
// get current CUDA device
int currentDevice;
error = cudaGetDevice(&currentDevice);
if (error != cudaSuccess) {
INSPIRE_LOGE("[CUDA error] failed to get current CUDA device: %s", cudaGetErrorString(error));
return HERR_DEVICE_CUDA_UNKNOWN_ERROR;
}
INSPIRE_LOGW("current CUDA device ID: %d", currentDevice);
// get GPU device properties
cudaDeviceProp prop;
error = cudaGetDeviceProperties(&prop, currentDevice);
if (error != cudaSuccess) {
INSPIRE_LOGE("[CUDA error] failed to get CUDA device properties: %s", cudaGetErrorString(error));
return HERR_DEVICE_CUDA_UNKNOWN_ERROR;
}
// print device detailed information
INSPIRE_LOGW("\nCUDA device details:");
INSPIRE_LOGW("device name: %s", prop.name);
INSPIRE_LOGW("compute capability: %d.%d", prop.major, prop.minor);
INSPIRE_LOGW("global memory: %d MB", prop.totalGlobalMem / (1024 * 1024));
INSPIRE_LOGW("max shared memory/block: %d KB", prop.sharedMemPerBlock / 1024);
INSPIRE_LOGW("max threads/block: %d", prop.maxThreadsPerBlock);
INSPIRE_LOGW("max block dimensions: (%d, %d, %d)", prop.maxThreadsDim[0], prop.maxThreadsDim[1], prop.maxThreadsDim[2]);
INSPIRE_LOGW("max grid size: (%d, %d, %d)", prop.maxGridSize[0], prop.maxGridSize[1], prop.maxGridSize[2]);
INSPIRE_LOGW("total constant memory: %d KB", prop.totalConstMem / 1024);
INSPIRE_LOGW("multi-processor count: %d", prop.multiProcessorCount);
INSPIRE_LOGW("max blocks per multi-processor: %d", prop.maxBlocksPerMultiProcessor);
INSPIRE_LOGW("clock frequency: %d MHz", prop.clockRate / 1000);
INSPIRE_LOGW("memory frequency: %d MHz", prop.memoryClockRate / 1000);
INSPIRE_LOGW("memory bus width: %d bits", prop.memoryBusWidth);
INSPIRE_LOGW("L2 cache size: %d KB", prop.l2CacheSize / 1024);
INSPIRE_LOGW("theoretical memory bandwidth: %f GB/s", 2.0 * prop.memoryClockRate * (prop.memoryBusWidth / 8) / 1.0e6);
// check if FP16 is supported
bool supportsFP16 = prop.major >= 6 || (prop.major == 5 && prop.minor >= 3);
INSPIRE_LOGW("FP16 support: %s", supportsFP16 ? "yes" : "no");
// check if unified memory is supported
INSPIRE_LOGW("unified memory support: %s", prop.unifiedAddressing ? "yes" : "no");
// check if concurrent kernel execution is supported
INSPIRE_LOGW("concurrent kernel execution: %s", prop.concurrentKernels ? "yes" : "no");
// check if asynchronous engine is supported
INSPIRE_LOGW("asynchronous engine count: %d", prop.asyncEngineCount);
return HSUCCEED;
} catch (const std::exception &e) {
INSPIRE_LOGE("error when printing CUDA device info: %s", e.what());
return HERR_DEVICE_CUDA_UNKNOWN_ERROR;
}
}
inline static int32_t PrintCudaDeviceInfo() {
INSPIRE_LOGW("================================================");
auto ret = _PrintCudaDeviceInfo();
INSPIRE_LOGW("================================================");
return ret;
}
} // namespace inspire
#endif // INSPIRE_CUDA_TOOLKIT_H
#endif // ISF_ENABLE_TENSORRT

25
cpp-package/inspireface/cpp/inspireface/middleware/inference_adapter/inference_adapter.h
View File

@@ -9,14 +9,13 @@
class XOutputData {
public:
XOutputData() : size(0), data(nullptr) {}
std::vector<float> CopyToFloatArray() {
if (!buffer.empty()) {
return buffer;
}
std::vector<float> floatArray;
floatArray.resize(size);
std::memcpy(floatArray.data(), data, size * sizeof(float));
@@ -25,7 +24,7 @@ public:
public:
size_t size;
float *data; // Use pointer
float* data; // Use pointer
std::vector<float> buffer; // Use copy
};
@@ -33,7 +32,6 @@ typedef std::vector<XOutputData> XOutputDataList;
class XTransform {
public:
XTransform() : swap_color(false) {
std::fill(std::begin(normalize.mean), std::end(normalize.mean), 0.0f);
std::fill(std::begin(normalize.norm), std::end(normalize.norm), 1.0f);
@@ -57,10 +55,9 @@ public:
int32_t height;
int32_t width;
int32_t channel;
uint8_t *data;
uint8_t* data;
};
typedef enum {
xEngineMNN,
xEngineRKNN,
@@ -73,19 +70,18 @@ public:
xRetErr = -1,
};
typedef enum {
xDefaultCPU,
xMNNCuda,
} SpecialBackend;
typedef enum { xDefaultCPU, xMNNCuda, xCoreML } SpecialBackend;
public:
virtual ~InferenceAdapter() {};
virtual int32_t SetNumThreads(const int32_t num_threads) = 0;
virtual int32_t Initialize(const std::string& model_filename, const XTransform& transform, const std::string& input_name, const std::vector<std::string> &outputs_name) = 0;
virtual int32_t Initialize(char* model_buffer, int model_size, const std::string& input_name, const XTransform& transform, const std::vector<std::string> &outputs_name) = 0;
virtual int32_t Initialize(const std::string& model_filename, const XTransform& transform, const std::string& input_name,
const std::vector<std::string>& outputs_name) = 0;
virtual int32_t Initialize(char* model_buffer, int model_size, const std::string& input_name, const XTransform& transform,
const std::vector<std::string>& outputs_name) = 0;
virtual int32_t Finalize(void) = 0;
virtual int32_t SetInputsData(const std::vector<XInputData>& batch, ) = 0;
virtual int32_t Forward(std::vector<XOutputDataList> &batch_outputs) = 0;
virtual int32_t Forward(std::vector<XOutputDataList>& batch_outputs) = 0;
virtual int32_t ResizeInputs() = 0;
@@ -97,9 +93,6 @@ public:
protected:
EngineType engine_type_;
SpecialBackend special_backend_ = xDefaultCPU;
};
#endif // INSPIREFACE_OMNI_INFERENACE__

432
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper.cpp
View File

@@ -1,432 +0,0 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/*** Include ***/
/* for general */
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
/* for My modules */
#include "inference_helper_log.h"
#include "inference_helper.h"
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
#include "inference_helper_opencv.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_TFLITE) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU) || defined(INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU)
#include "inference_helper_tensorflow_lite.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
#include "inference_helper_tensorrt.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_NCNN
#include "inference_helper_ncnn.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_MNN
#include "inference_helper_mnn.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_SNPE
#include "inference_helper_snpe.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
#include "inference_helper_armnn.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_NNABLA) || defined(INFERENCE_HELPER_ENABLE_NNABLA_CUDA)
#include "inference_helper_nnabla.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME) || defined(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA)
#include "inference_helper_onnx_runtime.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_LIBTORCH) || defined(INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA)
#include "inference_helper_libtorch.h"
#endif
#if defined(INFERENCE_HELPER_ENABLE_TENSORFLOW) || defined(INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU)
#include "inference_helper_tensorflow.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
#include "inference_helper_sample.h"
#endif
#ifdef INFERENCE_HELPER_ENABLE_RKNN
//#include "inference_helper_rknn.h"
#include "inference_helper_rknn_adapter.h"
#endif
/*** Macro ***/
#define TAG "InferenceHelper"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceHelper* InferenceHelper::Create(const InferenceHelper::HelperType helper_type)
{
InferenceHelper* p = nullptr;
switch (helper_type) {
#ifdef INFERENCE_HELPER_ENABLE_OPENCV
case kOpencv:
case kOpencvGpu:
PRINT("Use OpenCV \n");
p = new InferenceHelperOpenCV();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE
case kTensorflowLite:
PRINT("Use TensorflowLite\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK
case kTensorflowLiteXnnpack:
PRINT("Use TensorflowLite XNNPACK Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU
case kTensorflowLiteGpu:
PRINT("Use TensorflowLite GPU Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_EDGETPU
case kTensorflowLiteEdgetpu:
PRINT("Use TensorflowLite EdgeTPU Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_NNAPI
case kTensorflowLiteNnapi:
PRINT("Use TensorflowLite NNAPI Delegate\n");
p = new InferenceHelperTensorflowLite();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORRT
case kTensorrt:
PRINT("Use TensorRT \n");
p = new InferenceHelperTensorRt();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NCNN
case kNcnn:
case kNcnnVulkan:
PRINT("Use NCNN\n");
p = new InferenceHelperNcnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_MNN
case kMnn:
// PRINT("Use General Inference\n");
p = new InferenceHelperMnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_SNPE
case kSnpe:
PRINT("Use SNPE\n");
p = new InferenceHelperSnpe();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ARMNN
case kArmnn:
PRINT("Use ARMNN\n");
p = new InferenceHelperArmnn();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NNABLA
case kNnabla:
PRINT("Use NNabla\n");
p = new InferenceHelperNnabla();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_NNABLA_CUDA
case kNnablaCuda:
PRINT("Use NNabla_CUDA\n");
p = new InferenceHelperNnabla();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME
case kOnnxRuntime:
PRINT("Use ONNX Runtime\n");
p = new InferenceHelperOnnxRuntime();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA
case kOnnxRuntimeCuda:
PRINT("Use ONNX Runtime_CUDA\n");
p = new InferenceHelperOnnxRuntime();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH
case kLibtorch:
PRINT("Use LibTorch\n");
p = new InferenceHelperLibtorch();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA
case kLibtorchCuda:
PRINT("Use LibTorch CUDA\n");
p = new InferenceHelperLibtorch();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW
case kTensorflow:
PRINT("Use TensorFlow\n");
p = new InferenceHelperTensorflow();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU
case kTensorflowGpu:
PRINT("Use TensorFlow GPU\n");
p = new InferenceHelperTensorflow();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_SAMPLE
case kSample:
PRINT("Do not use this. this is just a reference code\n");
p = new InferenceHelperSample();
break;
#endif
#ifdef INFERENCE_HELPER_ENABLE_RKNN
case kRknn:
// PRINT("Use Rknn\n")
// p = new InferenceHelperRKNN();
p = new InferenceHelperRknnAdapter();
break;
#endif
default:
PRINT_E("Unsupported inference helper type (%d)\n", helper_type)
break;
}
if (p == nullptr) {
PRINT_E("Failed to create inference helper\n")
} else {
p->helper_type_ = helper_type;
}
return p;
}
#ifdef INFERENCE_HELPER_ENABLE_PRE_PROCESS_BY_OPENCV
#include <opencv2/opencv.hpp>
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
{
/* Generate mat from original data */
cv::Mat img_src = cv::Mat(cv::Size(input_tensor_info.image_info.width, input_tensor_info.image_info.height), (input_tensor_info.image_info.channel == 3) ? CV_8UC3 : CV_8UC1, input_tensor_info.data);
/* Crop image */
if (input_tensor_info.image_info.width == input_tensor_info.image_info.crop_width && input_tensor_info.image_info.height == input_tensor_info.image_info.crop_height) {
/* do nothing */
} else {
img_src = img_src(cv::Rect(input_tensor_info.image_info.crop_x, input_tensor_info.image_info.crop_y, input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height));
}
/* Resize image */
if (input_tensor_info.image_info.crop_width == input_tensor_info.GetWidth() && input_tensor_info.image_info.crop_height == input_tensor_info.GetHeight()) {
/* do nothing */
} else {
cv::resize(img_src, img_src, cv::Size(input_tensor_info.GetWidth(), input_tensor_info.GetHeight()));
}
/* Convert color type */
if (input_tensor_info.image_info.channel == input_tensor_info.GetChannel()) {
if (input_tensor_info.image_info.channel == 3 && input_tensor_info.image_info.swap_color) {
cv::cvtColor(img_src, img_src, cv::COLOR_BGR2RGB);
}
} else if (input_tensor_info.image_info.channel == 3 && input_tensor_info.GetChannel() == 1) {
cv::cvtColor(img_src, img_src, (input_tensor_info.image_info.is_bgr) ? cv::COLOR_BGR2GRAY : cv::COLOR_RGB2GRAY);
} else if (input_tensor_info.image_info.channel == 1 && input_tensor_info.GetChannel() == 3) {
cv::cvtColor(img_src, img_src, cv::COLOR_GRAY2BGR);
}
if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) {
/* Normalize image */
if (input_tensor_info.GetChannel() == 3) {
#if 1
img_src.convertTo(img_src, CV_32FC3);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
#else
img_src.convertTo(img_src, CV_32FC3, 1.0 / 255);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.mean)), img_src);
cv::divide(img_src, cv::Scalar(cv::Vec<float, 3>(input_tensor_info.normalize.norm)), img_src);
#endif
} else {
#if 1
img_src.convertTo(img_src, CV_32FC1);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
cv::multiply(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
#else
img_src.convertTo(img_src, CV_32FC1, 1.0 / 255);
cv::subtract(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.mean)), img_src);
cv::divide(img_src, cv::Scalar(cv::Vec<float, 1>(input_tensor_info.normalize.norm)), img_src);
#endif
}
} else {
/* do nothing */
}
if (is_nchw) {
/* Convert to 4-dimensional Mat in NCHW */
img_src = cv::dnn::blobFromImage(img_src);
}
img_blob = img_src;
//memcpy(blobData, img_src.data, img_src.cols * img_src.rows * img_src.channels());
}
#else
/* For the environment where OpenCV is not supported */
void InferenceHelper::PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob)
{
PRINT_E("[PreProcessByOpenCV] Unsupported function called\n");
exit(-1);
}
#endif
void InferenceHelper::ConvertNormalizeParameters(InputTensorInfo& tensor_info)
{
if (tensor_info.data_type != InputTensorInfo::kDataTypeImage) return;
#if 0
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = src * 1 / (255 * norm) - (mean / norm) */
for (int32_t i = 0; i < 3; i++) {
tensor_info.normalize.mean[i] /= tensor_info.normalize.norm[i];
tensor_info.normalize.norm[i] *= 255.0f;
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
}
#endif
#if 1
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = (src - (mean * 255)) * (1 / (255 * norm)) */
for (int32_t i = 0; i < 3; i++) {
tensor_info.normalize.mean[i] *= 255.0f;
tensor_info.normalize.norm[i] *= 255.0f;
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
}
#endif
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
}
}
} else {
/* convert NHWC to NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
#if 1
dst[i * img_channel + c] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
#else
dst[i * img_channel + c] = (src[i * img_channel + c] / 255.0f - input_tensor_info.normalize.mean[c]) / input_tensor_info.normalize.norm[c];
#endif
}
}
}
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c];
}
}
} else {
/* convert NHWC to NHWC */
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
}
}
void InferenceHelper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c] - 128;
}
}
} else {
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[i * img_channel + c] - 128;
}
}
}
}
template<typename T>
void InferenceHelper::PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T* dst)
{
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
T* src = static_cast<T*>(input_tensor_info.data);
if ((input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw && input_tensor_info.is_nchw) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc && !input_tensor_info.is_nchw)) {
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) {
/* NCHW -> NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[c * (img_width * img_height) + i];
}
}
} else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
/* NHWC -> NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[c * (img_width * img_height) + i] = src[i * img_channel + c];
}
}
}
}
template void InferenceHelper::PreProcessBlob<float>(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
template void InferenceHelper::PreProcessBlob<int32_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int32_t* dst);
template void InferenceHelper::PreProcessBlob<int64_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int64_t* dst);
template void InferenceHelper::PreProcessBlob<uint8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
template void InferenceHelper::PreProcessBlob<int8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);

295
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper.h
View File

@@ -1,295 +0,0 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_
#define INFERENCE_HELPER_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
class TensorInfo {
public:
enum {
kTensorTypeNone,
kTensorTypeUint8,
kTensorTypeInt8,
kTensorTypeFp32,
kTensorTypeInt32,
kTensorTypeInt64,
};
public:
TensorInfo()
: name("")
, id(-1)
, tensor_type(kTensorTypeNone)
, is_nchw(true)
{}
~TensorInfo() {}
int32_t GetElementNum() const
{
int32_t element_num = 1;
for (const auto& dim : tensor_dims) {
element_num *= dim;
}
return element_num;
}
int32_t GetBatch() const
{
if (tensor_dims.size() <= 0) return -1;
return tensor_dims[0];
}
int32_t GetChannel() const
{
if (is_nchw) {
if (tensor_dims.size() <= 1) return -1;
return tensor_dims[1];
} else {
if (tensor_dims.size() <= 3) return -1;
return tensor_dims[3];
}
}
int32_t GetHeight() const
{
if (is_nchw) {
if (tensor_dims.size() <= 2) return -1;
return tensor_dims[2];
} else {
if (tensor_dims.size() <= 1) return -1;
return tensor_dims[1];
}
}
int32_t GetWidth() const
{
if (is_nchw) {
if (tensor_dims.size() <= 3) return -1;
return tensor_dims[3];
} else {
if (tensor_dims.size() <= 2) return -1;
return tensor_dims[2];
}
}
public:
std::string name; // [In] Set the name_ of tensor
int32_t id; // [Out] Do not modify (Used in InferenceHelper)
int32_t tensor_type; // [In] The type of tensor (e.g. kTensorTypeFp32)
std::vector<int32_t> tensor_dims; // InputTensorInfo: [In] The dimentions of tensor. (If empty at initialize, the size is updated from model info.)
// OutputTensorInfo: [Out] The dimentions of tensor is set from model information
bool is_nchw; // [IN] NCHW or NHWC
};
class InputTensorInfo : public TensorInfo {
public:
enum {
kDataTypeImage,
kDataTypeBlobNhwc, // data_ which already finished preprocess(color conversion, resize, normalize_, etc.)
kDataTypeBlobNchw,
};
public:
InputTensorInfo()
: data(nullptr)
, data_type(kDataTypeImage)
, image_info({ -1, -1, -1, -1, -1, -1, -1, true, false })
, normalize({ 0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f })
{}
InputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
: InputTensorInfo()
{
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~InputTensorInfo() {}
public:
void* data; // [In] Set the pointer to image/blob
int32_t data_type; // [In] Set the type of data_ (e.g. kDataTypeImage)
struct {
int32_t width;
int32_t height;
int32_t channel;
int32_t crop_x;
int32_t crop_y;
int32_t crop_width;
int32_t crop_height;
bool is_bgr; // used when channel == 3 (true: BGR, false: RGB)
bool swap_color;
} image_info; // [In] used when data_type_ == kDataTypeImage
struct {
float mean[3];
float norm[3];
} normalize; // [In] used when data_type_ == kDataTypeImage
};
class OutputTensorInfo : public TensorInfo {
public:
OutputTensorInfo()
: data(nullptr)
, quant({ 1.0f, 0 })
, data_fp32_(nullptr)
{}
OutputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true)
: OutputTensorInfo()
{
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~OutputTensorInfo() {
if (data_fp32_ != nullptr) {
delete[] data_fp32_;
}
}
float* GetDataAsFloat() { /* Returned pointer should be with const, but returning pointer without const is convenient to create cv::Mat */
if (tensor_type == kTensorTypeUint8 || tensor_type == kTensorTypeInt8) {
if (data_fp32_ == nullptr) {
data_fp32_ = new float[GetElementNum()];
}
if (tensor_type == kTensorTypeUint8) {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const uint8_t* val_uint8 = static_cast<const uint8_t*>(data);
float val_float = (val_uint8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
} else {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const int8_t* val_int8 = static_cast<const int8_t*>(data);
float val_float = (val_int8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
}
return data_fp32_;
} else if (tensor_type == kTensorTypeFp32) {
return static_cast<float*>(data);
} else {
return nullptr;
}
}
public:
void* data; // [Out] Pointer to the output data_
struct {
float scale;
int32_t zero_point;
} quant; // [Out] Parameters for dequantization (convert uint8 to float)
private:
float* data_fp32_;
};
namespace cv {
class Mat;
};
class InferenceHelper {
public:
enum {
kRetOk = 0,
kRetErr = -1,
};
typedef enum {
kDefaultCPU,
kMnnCuda,
} SpecialBackend;
typedef enum {
kOpencv,
kOpencvGpu,
kTensorflowLite,
kTensorflowLiteXnnpack,
kTensorflowLiteGpu,
kTensorflowLiteEdgetpu,
kTensorflowLiteNnapi,
kTensorrt,
kNcnn,
kNcnnVulkan,
kMnn,
kSnpe,
kArmnn,
kNnabla,
kNnablaCuda,
kOnnxRuntime,
kOnnxRuntimeCuda,
kLibtorch,
kLibtorchCuda,
kTensorflow,
kTensorflowGpu,
kSample,
kRknn,
} HelperType;
public:
static InferenceHelper* Create(const HelperType helper_type);
static void PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob); // use this if the selected inference engine doesn't support pre-process
public:
virtual ~InferenceHelper() {}
virtual int32_t SetNumThreads(const int32_t num_threads) = 0;
virtual int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) = 0;
virtual int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Finalize(void) = 0;
virtual int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
virtual int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t SetSpecialBackend(SpecialBackend backend) {
special_backend_ = backend;
return kRetOk;
};
virtual int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
virtual std::vector<std::string> GetInputNames() = 0;
protected:
void ConvertNormalizeParameters(InputTensorInfo& tensor_info);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
template<typename T>
void PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T *dst);
protected:
HelperType helper_type_;
SpecialBackend special_backend_ = kDefaultCPU;
};
#endif

45
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_log.h
View File

@@ -1,45 +0,0 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_LOG_
#define INFERENCE_HELPER_LOG_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#if defined(ANDROID) || defined(__ANDROID__)
#define CV_COLOR_IS_RGB
#include <android/log.h>
#define INFERENCE_HELPER_LOG_NDK_TAG "HyperLPR3-Native-Inference"
#define INFERENCE_HELPER_LOG_PRINT_(...) __android_log_print(ANDROID_LOG_INFO, INFERENCE_HELPER_LOG_NDK_TAG, __VA_ARGS__)
#else
#define INFERENCE_HELPER_LOG_PRINT_(...) printf(__VA_ARGS__)
#endif
#define INFERENCE_HELPER_LOG_PRINT(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
INFERENCE_HELPER_LOG_PRINT_("[" INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
} while(0);
#define INFERENCE_HELPER_LOG_PRINT_E(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \
INFERENCE_HELPER_LOG_PRINT_("[ERR: " INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \
} while(0);
#endif

60
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_mnn.h
View File

@@ -1,60 +0,0 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef INFERENCE_HELPER_MNN_
#define INFERENCE_HELPER_MNN_
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
/* for MNN */
#include <MNN/ImageProcess.hpp>
#include <MNN/Interpreter.hpp>
#include <MNN/AutoTime.hpp>
/* for My modules */
#include "inference_helper.h"
class InferenceHelperMnn : public InferenceHelper {
public:
InferenceHelperMnn();
~InferenceHelperMnn() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
private:
std::unique_ptr<MNN::Interpreter> net_;
MNN::Session* session_;
std::vector<std::unique_ptr<MNN::Tensor>> out_mat_list_;
int32_t num_threads_;
std::vector<std::string> input_names_;
};
#endif

161
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_rknn_adapter.cpp
View File

@@ -1,161 +0,0 @@
//
// Created by tunm on 2023/9/24.
//
#ifdef INFERENCE_HELPER_ENABLE_RKNN
#include <iostream>
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include "inference_helper_rknn_adapter.h"
#include "inference_helper_log.h"
#include "log.h"
/*** Macro ***/
#define TAG "InferenceHelperRknn"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceHelperRknnAdapter::InferenceHelperRknnAdapter() {
num_threads_ = 1;
}
InferenceHelperRknnAdapter::~InferenceHelperRknnAdapter() {
}
int32_t InferenceHelperRknnAdapter::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return kRetOk;
}
int32_t InferenceHelperRknnAdapter::SetCustomOps(const std::vector<std::pair<const char *, const void *>> &custom_ops) {
PRINT("[WARNING] This method is not supported\n")
return kRetOk;
}
int32_t InferenceHelperRknnAdapter::ParameterInitialization(std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
return kRetOk;
}
int32_t InferenceHelperRknnAdapter::Process(std::vector<OutputTensorInfo> &output_tensor_info_list) {
if (output_tensor_info_list[0].tensor_type == TensorInfo::kTensorTypeFp32) {
net_->setOutputsWantFloat(1);
// INSPIRE_LOGD("WANT FLOAT!");
}
auto ret = net_->RunModel();
if (ret != 0) {
INSPIRE_LOGE("Run model error.");
return kRetErr;
}
auto outputs_size = net_->GetOutputsNum();
// INSPIRE_LOGD("==================");
// auto f = net_->GetOutputData(0);
// for (int i = 0; i < 512; ++i) {
// std::cout << f[i] << ", ";
// }
// std::cout << std::endl;
//
// INSPIRE_LOGD("==================");
assert(outputs_size == output_tensor_info_list.size());
for (int index = 0; index < outputs_size; ++index) {
auto &output_tensor = output_tensor_info_list[index];
output_tensor.data = net_->GetOutputFlow(index);
auto dim = net_->GetOutputTensorSize(index);
output_tensor.tensor_dims.clear();
for (int i = 0; i < dim.size(); ++i) {
output_tensor.tensor_dims.push_back((int)dim[i]);
// INSPIRE_LOGE("dim: %d", dim[i]);
}
}
net_->ReleaseOutputs();
return kRetOk;
}
int32_t InferenceHelperRknnAdapter::PreProcess(const std::vector<InputTensorInfo> &input_tensor_info_list) {
for (int i = 0; i < input_tensor_info_list.size(); ++i) {
auto &input_tensor_info = input_tensor_info_list[i];
// cv::Mat mat(input_tensor_info.GetHeight(), input_tensor_info.GetWidth(), CV_8UC3, input_tensor_info.data);
// INSPIRE_LOGD("decode : %d", input_tensor_info.GetHeight());
// INSPIRE_LOGD("decode : %d", input_tensor_info.GetWidth());
// cv::Mat bgr;
// cv::cvtColor(mat, bgr, cv::COLOR_RGB2BGR);
// cv::imwrite("dec.jpg", bgr);
rknn_tensor_format fmt = RKNN_TENSOR_NHWC;
if (input_tensor_info.is_nchw) {
fmt = RKNN_TENSOR_NCHW;
} else {
fmt = RKNN_TENSOR_NHWC;
// INSPIRE_LOGD("NHWC!");
}
rknn_tensor_type type = RKNN_TENSOR_UINT8;
if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::kTensorTypeFp32) {
type = RKNN_TENSOR_FLOAT32;
} else if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::kTensorTypeUint8) {
type = RKNN_TENSOR_UINT8;
// INSPIRE_LOGD("UINT8!");
}
auto ret = net_->SetInputData(i, input_tensor_info.data, input_tensor_info.GetWidth(), input_tensor_info.GetHeight(), input_tensor_info.GetChannel(), type, fmt);
if (ret != 0) {
INSPIRE_LOGE("Set data error.");
return ret;
}
}
return kRetOk;
}
int32_t
InferenceHelperRknnAdapter::Initialize(const std::string &model_filename, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
INSPIRE_LOGE("NOT IMPL");
return 0;
}
int32_t InferenceHelperRknnAdapter::Initialize(char *model_buffer, int model_size,
std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
net_ = std::make_shared<RKNNAdapter>();
auto ret = net_->Initialize((unsigned char* )model_buffer, model_size);
if (ret != 0) {
INSPIRE_LOGE("Rknn init error.");
return kRetErr;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceHelperRknnAdapter::Finalize(void) {
if (net_ != nullptr) {
net_->Release();
}
return kRetOk;
}
std::vector<std::string> InferenceHelperRknnAdapter::GetInputNames() {
return std::vector<std::string>();
}
int32_t InferenceHelperRknnAdapter::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
// The function is not supported
return 0;
}
#endif // INFERENCE_HELPER_ENABLE_RKNN

46
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_rknn_adapter.h
View File

@@ -1,46 +0,0 @@
//
// Created by tunm on 2023/9/24.
//
#ifndef HYPERFACEREPO_INFERENCE_HELPER_RKNN_ADAPTER_H
#define HYPERFACEREPO_INFERENCE_HELPER_RKNN_ADAPTER_H
#ifdef INFERENCE_HELPER_ENABLE_RKNN
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
/* for My modules */
#include "inference_helper.h"
#include "customized/rknn_adapter.h"
class InferenceHelperRknnAdapter: public InferenceHelper {
public:
InferenceHelperRknnAdapter();
~InferenceHelperRknnAdapter() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
private:
std::shared_ptr<RKNNAdapter> net_;
int32_t num_threads_;
};
#endif // INFERENCE_HELPER_ENABLE_RKNN
#endif //HYPERFACEREPO_INFERENCE_HELPER_RKNN_ADAPTER_H

84
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/coreml/CoreMLAdapter.h Normal file
View File

@@ -0,0 +1,84 @@
/**
* Created by Jingyu Yan
* @date 2025-02-01
*/
#ifndef INSPIRE_FACE_COREML_ADAPTER_H
#define INSPIRE_FACE_COREML_ADAPTER_H
#include <string>
#include <vector>
#include <map>
#ifdef __OBJC__
@class MLModel;
@protocol MLFeatureProvider;
#else
typedef void MLModel;
typedef void MLFeatureProvider;
#endif
#define COREML_HSUCCEED 0
#define COREML_HFAIL -1
#define COREML_FORWARD_FAILED -2
/**
* @brief CoreML adapter for inference
*/
class CoreMLAdapter {
public:
/**
* @brief Inference mode
*/
enum class InferenceMode {
CPU, ///< CPU inference
GPU, ///< GPU inference
ANE ///< Automatic selection, ANE first
};
/**
* @brief Output shapes map
*/
typedef std::map<std::string, std::vector<int>> OutputShapesMap;
CoreMLAdapter();
~CoreMLAdapter();
/**
* @brief Read model from file
* @param modelPath model path
* @return 0 if success, -1 if failed
*/
int32_t readFromFile(const std::string &modelPath);
static CoreMLAdapter readNetFrom(const std::string &modelPath);
/**
* @brief Not implemented
*/
static CoreMLAdapter readNetFromBin(const std::vector<char> &model_data);
std::vector<std::string> getInputNames() const;
std::vector<std::string> getOutputNames() const;
std::vector<int> getInputShapeByName(const std::string &name);
std::vector<int> getOutputShapeByName(const std::string &name);
void setInput(const char *inputName, const char *data);
int32_t forward();
const char *getOutput(const char *nodeName);
const std::vector<int> &getOutputShapeByName(const std::string &name) const;
/**
* @brief Set inference mode
* @param mode inference mode
*/
void setInferenceMode(InferenceMode mode);
void printModelInfo() const;
private:
class Impl;
Impl *pImpl;
OutputShapesMap m_outputShapes;
};
#endif // INSPIRE_FACE_COREML_ADAPTER_H

278
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/coreml/CoreMLAdapter.mm Normal file
View File

@@ -0,0 +1,278 @@
#import "CoreMLAdapter.h"
#import <CoreML/CoreML.h>
#import <Foundation/Foundation.h>
#import <Accelerate/Accelerate.h>
@interface CoreMLAdapterImpl : NSObject
@property (nonatomic, strong) MLModel *model;
@property (nonatomic, strong) id<MLFeatureProvider> inputFeatures;
@property (nonatomic, strong) id<MLFeatureProvider> outputFeatures;
@property (nonatomic, strong) NSURL *modelURL;
@end
@implementation CoreMLAdapterImpl
@end // CoreMLAdapterImpl
class CoreMLAdapter::Impl {
public:
CoreMLAdapterImpl *impl;
};
CoreMLAdapter::CoreMLAdapter() : pImpl(new Impl) {
pImpl->impl = [[CoreMLAdapterImpl alloc] init];
}
CoreMLAdapter::~CoreMLAdapter() {
delete pImpl;
}
int32_t CoreMLAdapter::readFromFile(const std::string &modelPath) {
NSString *modelPathStr = [NSString stringWithUTF8String:modelPath.c_str()];
NSURL *modelURL = [NSURL fileURLWithPath:modelPathStr];
NSError *error = nil;
pImpl->impl.model = [MLModel modelWithContentsOfURL:modelURL error:&error];
pImpl->impl.modelURL = modelURL;
if (error) {
NSLog(@"Error loading model: %@", error);
return COREML_HFAIL;
}
// printModelInfo();
return COREML_HSUCCEED;
}
CoreMLAdapter CoreMLAdapter::readNetFrom(const std::string& modelPath) {
CoreMLAdapter net;
NSString *modelPathStr = [NSString stringWithUTF8String:modelPath.c_str()];
NSURL *modelURL = [NSURL fileURLWithPath:modelPathStr];
NSError *error = nil;
net.pImpl->impl.model = [MLModel modelWithContentsOfURL:modelURL error:&error];
net.pImpl->impl.modelURL = modelURL;
if (error) {
NSLog(@"Error loading model: %@", error);
}
return net;
}
CoreMLAdapter CoreMLAdapter::readNetFromBin(const std::vector<char>& model_data) {
// Currently, reading binary files directly is not supported!
CoreMLAdapter net;
NSLog(@"readNetFromBin not fully implemented");
return net;
}
std::vector<std::string> CoreMLAdapter::getInputNames() const {
std::vector<std::string> names;
for (NSString *name in pImpl->impl.model.modelDescription.inputDescriptionsByName.allKeys) {
names.push_back([name UTF8String]);
}
return names;
}
std::vector<std::string> CoreMLAdapter::getOutputNames() const {
std::vector<std::string> names;
for (NSString *name in pImpl->impl.model.modelDescription.outputDescriptionsByName.allKeys) {
names.push_back([name UTF8String]);
}
return names;
}
std::vector<int> CoreMLAdapter::getInputShapeByName(const std::string &name) {
NSString *nsName = [NSString stringWithUTF8String:name.c_str()];
MLFeatureDescription *desc = pImpl->impl.model.modelDescription.inputDescriptionsByName[nsName];
if (desc.type != MLFeatureTypeMultiArray) {
return {};
}
std::vector<int> shape;
for (NSNumber *dim in desc.multiArrayConstraint.shape) {
shape.push_back(dim.intValue);
}
return shape;
}
std::vector<int> CoreMLAdapter::getOutputShapeByName(const std::string &name) {
if (m_outputShapes.find(name) != m_outputShapes.end()) {
return m_outputShapes[name];
}
return {};
}
void CoreMLAdapter::setInput(const char* inputName, const char* data) {
NSString *nsInputName = [NSString stringWithUTF8String:inputName];
MLFeatureDescription *desc = pImpl->impl.model.modelDescription.inputDescriptionsByName[nsInputName];
if (desc.type != MLFeatureTypeMultiArray) {
NSLog(@"Input %s is not a MultiArray", inputName);
return;
}
NSArray<NSNumber *> *shape = desc.multiArrayConstraint.shape;
NSMutableArray<NSNumber *> *strides = [NSMutableArray arrayWithCapacity:shape.count];
int stride = 1;
for (NSInteger i = shape.count - 1; i >= 0; i--) {
[strides insertObject:@(stride) atIndex:0];
stride *= [shape[i] intValue];
}
NSError *error = nil;
MLMultiArray *multiArray = [[MLMultiArray alloc] initWithDataPointer:(void*)data
shape:shape
dataType:MLMultiArrayDataTypeFloat32
strides:strides
deallocator:nil
error:&error];
if (error) {
NSLog(@"Error creating MLMultiArray: %@", error);
return;
}
MLFeatureValue *featureValue = [MLFeatureValue featureValueWithMultiArray:multiArray];
pImpl->impl.inputFeatures = [[MLDictionaryFeatureProvider alloc] initWithDictionary:@{nsInputName: featureValue} error:&error];
if (error) {
NSLog(@"Error creating MLDictionaryFeatureProvider: %@", error);
}
}
int32_t CoreMLAdapter::forward() {
@autoreleasepool {
if (!pImpl->impl.inputFeatures) {
NSLog(@"Input features not set");
return COREML_FORWARD_FAILED;
}
NSError *error = nil;
pImpl->impl.outputFeatures = [pImpl->impl.model predictionFromFeatures:pImpl->impl.inputFeatures
options:[[MLPredictionOptions alloc] init]
error:&error];
if (error) {
NSLog(@"Error in forward pass: %@", error);
return COREML_FORWARD_FAILED;
}
m_outputShapes.clear();
for (NSString *outputName in [pImpl->impl.outputFeatures featureNames]) {
MLFeatureValue *value = [pImpl->impl.outputFeatures featureValueForName:outputName];
if (value.multiArrayValue) {
NSArray<NSNumber *> *shapeArray = value.multiArrayValue.shape;
std::vector<int> shapeVector;
for (NSNumber *dim in shapeArray) {
shapeVector.push_back([dim intValue]);
}
std::string outputNameStr = [outputName UTF8String];
m_outputShapes[outputNameStr] = shapeVector;
}
}
return COREML_HSUCCEED;
}
}
const char* CoreMLAdapter::getOutput(const char* nodeName) {
@autoreleasepool {
if (!pImpl->impl.outputFeatures) {
NSLog(@"No output features available. Did you call forward()?");
return nullptr;
}
NSString *nsNodeName = [NSString stringWithUTF8String:nodeName];
MLFeatureValue *featureValue = [pImpl->impl.outputFeatures featureValueForName:nsNodeName];
if (featureValue.type != MLFeatureTypeMultiArray) {
NSLog(@"Error getting output for node '%s': Output is not a MultiArray. Current type: %@", nodeName, @(featureValue.type));
return nullptr;
}
return (const char*)featureValue.multiArrayValue.dataPointer;
}
}
void CoreMLAdapter::setInferenceMode(InferenceMode mode) {
MLComputeUnits computeUnits;
switch (mode) {
case InferenceMode::CPU:
computeUnits = MLComputeUnitsCPUOnly;
break;
case InferenceMode::GPU:
computeUnits = MLComputeUnitsCPUAndGPU;
break;
case InferenceMode::ANE:
computeUnits = MLComputeUnitsAll;
break;
}
MLModelConfiguration *config = [[MLModelConfiguration alloc] init];
config.computeUnits = computeUnits;
NSError *error = nil;
pImpl->impl.model = [MLModel modelWithContentsOfURL:pImpl->impl.modelURL
configuration:config
error:&error];
if (error) {
NSLog(@"Error setting inference mode: %@", error);
}
}
void CoreMLAdapter::printModelInfo() const {
NSLog(@"Model Input Description:");
for (NSString *inputName in pImpl->impl.model.modelDescription.inputDescriptionsByName) {
MLFeatureDescription *desc = pImpl->impl.model.modelDescription.inputDescriptionsByName[inputName];
NSLog(@"Input Name: %@", inputName);
NSLog(@"Input Type: %@", @(desc.type));
switch (desc.type) {
case MLFeatureTypeMultiArray:
NSLog(@"MLFeatureTypeMultiArray");
NSLog(@" Shape: %@", desc.multiArrayConstraint.shape);
NSLog(@" Data Type: %@", @(desc.multiArrayConstraint.dataType));
break;
case MLFeatureTypeImage:
NSLog(@"MLFeatureTypeImage");
NSLog(@" Image Size: %dx%d", (int)desc.imageConstraint.pixelsWide, (int)desc.imageConstraint.pixelsHigh);
NSLog(@" Color Space: %@", desc.imageConstraint.pixelFormatType == kCVPixelFormatType_32BGRA ? @"BGRA" : @"Other");
break;
case MLFeatureTypeDictionary:
NSLog(@" Dictionary Key Type: %@", @(desc.dictionaryConstraint.keyType));
break;
case MLFeatureTypeSequence:
NSLog(@" Sequence Constraint: %@", desc.sequenceConstraint);
break;
default:
NSLog(@" Unknown type details");
break;
}
}
NSLog(@"Model Output Description:");
for (NSString *outputName in pImpl->impl.model.modelDescription.outputDescriptionsByName) {
MLFeatureDescription *desc = pImpl->impl.model.modelDescription.outputDescriptionsByName[outputName];
NSLog(@"Output Name: %@", outputName);
NSLog(@"Output Type: %@", @(desc.type));
switch (desc.type) {
case MLFeatureTypeMultiArray:
NSLog(@" Shape: %@", desc.multiArrayConstraint.shape);
NSLog(@" Data Type: %@", @(desc.multiArrayConstraint.dataType));
break;
case MLFeatureTypeImage:
NSLog(@" Image Size: %dx%d", (int)desc.imageConstraint.pixelsWide, (int)desc.imageConstraint.pixelsHigh);
NSLog(@" Color Space: %@", desc.imageConstraint.pixelFormatType == kCVPixelFormatType_32BGRA ? @"BGRA" : @"Other");
break;
case MLFeatureTypeDictionary:
NSLog(@" Dictionary Key Type: %@", @(desc.dictionaryConstraint.keyType));
break;
case MLFeatureTypeSequence:
NSLog(@" Sequence Constraint: %@", desc.sequenceConstraint);
break;
default:
NSLog(@" Unknown type details");
break;
}
}
}

148
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/customized/rknn_adapter.h → cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/customized/rknn_adapter.h
View File

@@ -1,17 +1,16 @@
//
// Created by Tunm-Air13 on 2022/10/10.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef MAGIC_GESTURES_RKNN_ADAPTER_H
#define MAGIC_GESTURES_RKNN_ADAPTER_H
#ifndef INSPIREFACE_RKNN_ADAPTER_RKNPU1_H
#define INSPIREFACE_RKNN_ADAPTER_RKNPU1_H
#include <iostream>
#include "opencv2/opencv.hpp"
#include "rknn_api.h"
#include "data_type.h"
#include "log.h"
/**
* @brief Function to get RKNN data type string.
* @param type Data type
@@ -92,7 +91,7 @@ inline unsigned char *load_data_(FILE *fp, size_t ofst, size_t sz) {
return NULL;
}
data = (unsigned char *) malloc(sz);
data = (unsigned char *)malloc(sz);
if (data == NULL) {
printf("buffer malloc failure.\n");
return NULL;
@@ -133,9 +132,9 @@ inline unsigned char *load_model_(const char *filename, int *model_size) {
* @ingroup NeuralNetwork
*/
enum Status {
SUCCESS = 0, ///< Executed successfully
ERROR_SHAPE_MATCH = 1, ///< Execution error. tensor shapes don't match
ERROR_DATA_ORDER = 2 ///< Execution error, tensor data sorting error
SUCCESS = 0, ///< Executed successfully
ERROR_SHAPE_MATCH = 1, ///< Execution error. tensor shapes don't match
ERROR_DATA_ORDER = 2 ///< Execution error, tensor data sorting error
};
/**
@@ -145,7 +144,6 @@ enum Status {
*/
class RKNNAdapter {
public:
RKNNAdapter(const RKNNAdapter &) = delete;
RKNNAdapter &operator=(const RKNNAdapter &) = delete;
RKNNAdapter() = default;
@@ -162,7 +160,7 @@ public:
model_data = load_model_(model_path, &model_data_size);
load_ = true;
int ret = rknn_init(&rk_ctx_, model_data, model_data_size, 0);
// INSPIRE_LOG_INFO("RKNN Init ok.");
// INSPIRE_LOG_INFO("RKNN Init ok.");
if (ret < 0) {
INSPIRE_LOGE("rknn_init fail! ret=%d", ret);
return -1;
@@ -172,7 +170,6 @@ public:
return init_();
}
/**
* @brief Manually initialize
* @details Initialize the RKNN model using model data and its size, and allocate memory for creating the inference engine session
@@ -180,10 +177,10 @@ public:
* @param model_size Size of the model data
* @return Initialization result
*/
int Initialize(const unsigned char* model_data, const unsigned int model_size) {
int Initialize(const unsigned char *model_data, const unsigned int model_size) {
/* Create the neural network */
INSPIRE_LOGD("The neural network is being initialized...");
int ret = rknn_init(&rk_ctx_, (void *) model_data, model_size, 0);
int ret = rknn_init(&rk_ctx_, (void *)model_data, model_size, 0);
if (ret < 0) {
INSPIRE_LOGE("rknn_init fail! ret=%d", ret);
@@ -200,9 +197,7 @@ public:
* @return Dimensions information composed of various sizes
*/
std::vector<int> GetInputTensorSize(const int &index) {
std::vector<int> dims(input_attrs_[index].dims,
input_attrs_[index].dims +
input_attrs_[index].n_dims);
std::vector<int> dims(input_attrs_[index].dims, input_attrs_[index].dims + input_attrs_[index].n_dims);
return dims;
}
@@ -212,10 +207,8 @@ public:
* @return Dimensions information composed of various sizes
*/
std::vector<unsigned long> GetOutputTensorSize(const int &index) {
// std::cout << "output_attrs_[index].n_dims:" << output_attrs_[index].n_dims << std::endl;
std::vector<unsigned long> dims(output_attrs_[index].dims,
output_attrs_[index].dims +
output_attrs_[index].n_dims);
// std::cout << "output_attrs_[index].n_dims:" << output_attrs_[index].n_dims << std::endl;
std::vector<unsigned long> dims(output_attrs_[index].dims, output_attrs_[index].dims + output_attrs_[index].n_dims);
return dims;
}
@@ -227,33 +220,11 @@ public:
int GetOutputTensorLen(const int &index) {
std::vector<unsigned long> tensor_size_out = GetOutputTensorSize(index);
int size = 1;
for (auto &one: tensor_size_out) size *= one;
for (auto &one : tensor_size_out)
size *= one;
return size;
}
/**
* @brief Set the data stream for the input layer
* @param index Index of the input layer
* @param data Image data in the form of an OpenCV Mat
* @return Input status
*/
Status SetInputData(const int index, const cv::Mat &data) {
if (data.type() != CV_8UC3) {
INSPIRE_LOGE("error: input data required CV_8UC3");
}
if (index < input_tensors_.size()) {
input_tensors_[index].index = 0;
input_tensors_[index].type = RKNN_TENSOR_UINT8;
input_tensors_[index].size = data.cols * data.rows * data.channels();
input_tensors_[index].fmt = RKNN_TENSOR_NHWC;
input_tensors_[index].buf = data.data;
input_tensors_[index].pass_through = 0;
} else {
INSPIRE_LOGE("error: assert index < len");
}
return SUCCESS;
}
/**
* @brief Set the data stream for the input layer
* @param index Index of the input layer
@@ -265,8 +236,7 @@ public:
* @param format Format of the input data (default: RKNN_TENSOR_NHWC)
* @return Input status
*/
Status SetInputData(const int index, void* data, int width, int height, int channels,
rknn_tensor_type type = RKNN_TENSOR_UINT8,
Status SetInputData(const int index, void *data, int width, int height, int channels, rknn_tensor_type type = RKNN_TENSOR_UINT8,
rknn_tensor_format format = RKNN_TENSOR_NHWC) {
if (index < input_tensors_.size()) {
input_tensors_[index].index = 0;
@@ -287,7 +257,7 @@ public:
* @return Inference status result
*/
int RunModel() {
// INSPIRE_LOGD("set input");
// INSPIRE_LOGD("set input");
int ret = rknn_inputs_set(rk_ctx_, rk_io_num_.n_input, input_tensors_.data());
if (ret < 0)
INSPIRE_LOGE("rknn_input fail! ret=%d", ret);
@@ -296,7 +266,7 @@ public:
output_tensors_[i].want_float = outputs_want_float_;
}
// INSPIRE_LOGD("rknn_run");
// INSPIRE_LOGD("rknn_run");
ret = rknn_run(rk_ctx_, nullptr);
if (ret < 0) {
INSPIRE_LOGE("rknn_run fail! ret=%d", ret);
@@ -317,7 +287,7 @@ public:
* @return Pointer to the output data
*/
const float *GetOutputData(const int index) {
return (float *) (output_tensors_[index].buf);
return (float *)(output_tensors_[index].buf);
}
/**
@@ -336,7 +306,7 @@ public:
* @return Returns a pointer to the output data
*/
u_int8_t *GetOutputDataU8(const int index) {
return (uint8_t *) (output_tensors_[index].buf);
return (uint8_t *)(output_tensors_[index].buf);
}
int32_t ReleaseOutputs() {
@@ -350,8 +320,7 @@ public:
* @param index_name Name of the input tensor
* @param shape New shape for the input tensor
*/
void ResizeInputTensor(const std::string &index_name,
const std::vector<int> &shape) {
void ResizeInputTensor(const std::string &index_name, const std::vector<int> &shape) {
// No implementation
}
@@ -375,7 +344,7 @@ public:
/**
* @brief Get a reference to the vector of output tensors
* @return A reference to the vector of output tensors.
*/
*/
std::vector<rknn_output> &GetOutputTensors() {
return output_tensors_;
}
@@ -398,7 +367,7 @@ public:
* @details Release all resources in memory, typically called in the destructor
*/
void Release() {
if (run_){
if (run_) {
rknn_destroy(rk_ctx_);
if (load_) {
free(model_data);
@@ -430,8 +399,7 @@ private:
}
INSPIRE_LOGD("sdk version: %s driver version: %s", version.api_version, version.drv_version);
ret = rknn_query(rk_ctx_, RKNN_QUERY_IN_OUT_NUM, &rk_io_num_,
sizeof(rk_io_num_));
ret = rknn_query(rk_ctx_, RKNN_QUERY_IN_OUT_NUM, &rk_io_num_, sizeof(rk_io_num_));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query ctx fail! ret=%d", ret);
@@ -440,8 +408,7 @@ private:
INSPIRE_LOGD("models input num: %d, output num: %d", rk_io_num_.n_input, rk_io_num_.n_output);
// spdlog::trace("input tensors: ");
// spdlog::trace("input tensors: ");
input_attrs_.resize(rk_io_num_.n_input);
output_attrs_.resize(rk_io_num_.n_output);
input_tensors_.resize(rk_io_num_.n_input);
@@ -451,8 +418,7 @@ private:
memset(&input_attrs_[i], 0, sizeof(input_attrs_[i]));
memset(&input_tensors_[i], 0, sizeof(input_tensors_[i]));
input_attrs_[i].index = i;
ret = rknn_query(rk_ctx_, RKNN_QUERY_INPUT_ATTR, &(input_attrs_[i]),
sizeof(rknn_tensor_attr));
ret = rknn_query(rk_ctx_, RKNN_QUERY_INPUT_ATTR, &(input_attrs_[i]), sizeof(rknn_tensor_attr));
INSPIRE_LOGD("input node index %d", i);
int channel = 3;
@@ -468,39 +434,36 @@ private:
height = input_attrs_[i].dims[2];
}
INSPIRE_LOGD("models input height=%d, width=%d, channel=%d", height, width, channel);
// print_tensor_attr_(input_attrs_);
// print_tensor_attr_(input_attrs_);
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret=%d", ret);
return -1;
}
}
// printf("[debug]models input num: %d, output num: %d\n", rk_io_num_.n_input, rk_io_num_.n_output);
// printf("[debug]models input num: %d, output num: %d\n", rk_io_num_.n_input, rk_io_num_.n_output);
for (int i = 0; i < rk_io_num_.n_output; ++i) {
memset(&output_attrs_[i], 0, sizeof(output_attrs_[i]));
memset(&output_tensors_[i], 0, sizeof(output_tensors_[i]));
output_attrs_[i].index = i;
ret = rknn_query(rk_ctx_, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs_[i]),
sizeof(rknn_tensor_attr));
ret = rknn_query(rk_ctx_, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs_[i]), sizeof(rknn_tensor_attr));
if (output_attrs_[i].qnt_type != RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC ||
output_attrs_[i].type != RKNN_TENSOR_UINT8) {
INSPIRE_LOGW("The Demo required for a Affine asymmetric u8 quantized rknn models, but output quant type is %s, output "
"data type is %s",
get_qnt_type_string_(output_attrs_[i].qnt_type), get_type_string_(output_attrs_[i].type));
// return -1;
if (output_attrs_[i].qnt_type != RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC || output_attrs_[i].type != RKNN_TENSOR_UINT8) {
INSPIRE_LOGW(
"The Demo required for a Affine asymmetric u8 quantized rknn models, but output quant type is %s, output "
"data type is %s",
get_qnt_type_string_(output_attrs_[i].qnt_type), get_type_string_(output_attrs_[i].type));
// return -1;
}
// print_tensor_attr_(output_attrs_[i]);
// rknn_tensor_attr rknn_attr;
// memset(&rknn_attr, 0, sizeof(rknn_tensor_attr));
//
// ret = rknn_query(rk_ctx_, RKNN_QUERY_OUTPUT_ATTR, &rknn_attr,
// sizeof(rknn_tensor_attr));
// printf("output node index %d \n", i);
// print_tensor_attr_(rknn_attr);
// print_tensor_attr_(output_attrs_[i]);
// rknn_tensor_attr rknn_attr;
// memset(&rknn_attr, 0, sizeof(rknn_tensor_attr));
//
// ret = rknn_query(rk_ctx_, RKNN_QUERY_OUTPUT_ATTR, &rknn_attr,
// sizeof(rknn_tensor_attr));
// printf("output node index %d \n", i);
// print_tensor_attr_(rknn_attr);
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret=%d", ret);
@@ -512,25 +475,24 @@ private:
}
private:
rknn_context rk_ctx_; ///< The context manager for RKNN.
rknn_input_output_num rk_io_num_; ///< The number of input and output streams in RKNN.
rknn_context rk_ctx_; ///< The context manager for RKNN.
rknn_input_output_num rk_io_num_; ///< The number of input and output streams in RKNN.
std::vector<rknn_tensor_attr> input_attrs_; ///< Attributes of input tensors.
std::vector<rknn_tensor_attr> output_attrs_; ///< Attributes of output tensors.
std::vector<rknn_input> input_tensors_; ///< Input data for the neural network.
std::vector<rknn_output> output_tensors_; ///< Output data from the neural network.
int outputs_want_float_ = 0; ///< Flag to indicate support for floating-point output.
int outputs_want_float_ = 0; ///< Flag to indicate support for floating-point output.
std::vector<int> tensor_shape_; ///< The shape of input tensors.
int width_; ///< The width of input data (typically for images).
int height_; ///< The height of input data (typically for images).
bool run_status_; ///< Flag to indicate the execution status of the neural network.
std::vector<int> tensor_shape_; ///< The shape of input tensors.
int width_; ///< The width of input data (typically for images).
int height_; ///< The height of input data (typically for images).
bool run_status_; ///< Flag to indicate the execution status of the neural network.
unsigned char *model_data; ///< Pointer to the model's data stream.
unsigned char *model_data; ///< Pointer to the model's data stream.
bool load_;
bool run_;
};
#endif //MAGIC_GESTURES_RKNN_ADAPTER_H
#endif // INSPIREFACE_RKNN_ADAPTER_RKNPU1_H

312
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/customized/rknn_adapter_nano.h Normal file
View File

@@ -0,0 +1,312 @@
//
// Created by Tunm-Air13 on 2023/11/3.
//
#ifndef SLEEPMONITORING_RKNN_ADAPTER_NANO_H
#define SLEEPMONITORING_RKNN_ADAPTER_NANO_H
#include "log.h"
#include "memory"
#include <float.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include "rknn_api.h"
#include <algorithm>
#include <iostream>
#include <cmath>
#include <vector>
#include "log.h"
inline std::vector<float> softmax(const std::vector<float> &input) {
std::vector<float> output;
output.reserve(input.size());
float max = *std::max_element(input.begin(), input.end());
float sum = 0.0;
for (float val : input) {
sum += std::exp(val - max);
}
for (float val : input) {
output.push_back(std::exp(val - max) / sum);
}
return output;
}
inline int NC1HWC2_int8_to_NCHW_float(const int8_t *src, float *dst, int *dims, int channel, int h, int w, int zp, float scale) {
int batch = dims[0];
int C1 = dims[1];
int C2 = dims[4];
int hw_src = dims[2] * dims[3];
int hw_dst = h * w;
for (int i = 0; i < batch; i++) {
src = src + i * C1 * hw_src * C2;
dst = dst + i * channel * hw_dst;
for (int c = 0; c < channel; ++c) {
int plane = c / C2;
const int8_t *src_c = plane * hw_src * C2 + src;
int offset = c % C2;
for (int cur_h = 0; cur_h < h; ++cur_h)
for (int cur_w = 0; cur_w < w; ++cur_w) {
int cur_hw = cur_h * w + cur_w;
dst[c * hw_dst + cur_h * w + cur_w] = (src_c[C2 * cur_hw + offset] - zp) * scale; // int8-->float
}
}
}
return 0;
}
static void dump_tensor_attr(rknn_tensor_attr *attr) {
char dims[128] = {0};
for (int i = 0; i < attr->n_dims; ++i) {
int idx = strlen(dims);
sprintf(&dims[idx], "%d%s", attr->dims[i], (i == attr->n_dims - 1) ? "" : ", ");
}
INSPIRE_LOGD(
" index=%d, name=%s, n_dims=%d, dims=[%s], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, "
"zp=%d, scale=%f",
attr->index, attr->name, attr->n_dims, dims, attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type),
get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale);
}
class RKNNAdapterNano {
public:
RKNNAdapterNano(const RKNNAdapterNano &) = delete;
RKNNAdapterNano &operator=(const RKNNAdapterNano &) = delete;
RKNNAdapterNano() = default;
int32_t Initialize(void *model_data, unsigned int model_size) {
int ret = rknn_init(&m_rk_ctx_, model_data, model_size, 0, NULL);
if (ret < 0) {
INSPIRE_LOGE("rknn_init fail! ret = %d", ret);
return -1;
}
// Get sdk and driver version
rknn_sdk_version sdk_ver;
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_SDK_VERSION, &sdk_ver, sizeof(sdk_ver));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret = %d", ret);
return -1;
}
INSPIRE_LOGD("rknn_api/rknnrt version: %s, driver version: %s", sdk_ver.api_version, sdk_ver.drv_version);
// Get Model Input Output Info
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_IN_OUT_NUM, &m_rk_io_num_, sizeof(m_rk_io_num_));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret = %d", ret);
return -1;
}
INSPIRE_LOGD("model input num: %d, output num: %d", m_rk_io_num_.n_input, m_rk_io_num_.n_output);
INSPIRE_LOGD("input tensors:");
m_input_attrs_.resize(m_rk_io_num_.n_input);
for (uint32_t i = 0; i < m_rk_io_num_.n_input; i++) {
memset(&m_input_attrs_[i], 0, sizeof(m_input_attrs_[i]));
m_input_attrs_[i].index = i;
// query info
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_INPUT_ATTR, &(m_input_attrs_[i]), sizeof(rknn_tensor_attr));
if (ret < 0) {
INSPIRE_LOGE("rknn_init error! ret = %d", ret);
return -1;
}
dump_tensor_attr(&m_input_attrs_[i]);
}
INSPIRE_LOGD("output tensors:");
m_output_attrs_.resize(m_rk_io_num_.n_output);
for (uint32_t i = 0; i < m_rk_io_num_.n_output; i++) {
memset(&m_output_attrs_[i], 0, sizeof(m_output_attrs_[i]));
m_output_attrs_[i].index = i;
// query info
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_NATIVE_NHWC_OUTPUT_ATTR, &(m_output_attrs_[i]), sizeof(rknn_tensor_attr));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret = %d", ret);
return -1;
}
dump_tensor_attr(&m_output_attrs_[i]);
}
// Get custom string
rknn_custom_string custom_string;
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_CUSTOM_STRING, &custom_string, sizeof(custom_string));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret = %d", ret);
return -1;
}
INSPIRE_LOGD("custom string: %s", custom_string.string);
// Create input tensor memory
m_input_mems_.resize(m_rk_io_num_.n_input);
for (int i = 0; i < m_rk_io_num_.n_input; ++i) {
m_input_mems_[i] = rknn_create_mem(m_rk_ctx_, m_input_attrs_[i].size_with_stride);
}
// Create output tensor memory
m_output_mems_.resize(m_rk_io_num_.n_output);
for (int i = 0; i < m_rk_io_num_.n_output; ++i) {
m_output_mems_[i] = rknn_create_mem(m_rk_ctx_, m_output_attrs_[i].size_with_stride);
}
INSPIRE_LOGD("output origin tensors:");
m_orig_output_attrs_.resize(m_rk_io_num_.n_output);
for (uint32_t i = 0; i < m_rk_io_num_.n_output; i++) {
memset(&m_orig_output_attrs_[i], 0, sizeof(m_orig_output_attrs_[i]));
m_orig_output_attrs_[i].index = i;
// query info
ret = rknn_query(m_rk_ctx_, RKNN_QUERY_OUTPUT_ATTR, &(m_orig_output_attrs_[i]), sizeof(rknn_tensor_attr));
if (ret != RKNN_SUCC) {
INSPIRE_LOGE("rknn_query fail! ret = %d", ret);
return -1;
}
dump_tensor_attr(&m_orig_output_attrs_[i]);
}
run_ = true;
return 0;
}
int32_t SetInputData(const int index, uint8_t *data, rknn_tensor_type type = RKNN_TENSOR_UINT8, rknn_tensor_format format = RKNN_TENSOR_NHWC) {
if (index < m_input_mems_.size()) {
m_input_attrs_[index].type = type;
m_input_attrs_[index].fmt = format;
// Copy input data to input tensor memory
int width = m_input_attrs_[index].dims[2];
int stride = m_input_attrs_[index].w_stride;
if (width == stride) {
memcpy(m_input_mems_[index]->virt_addr, data, width * m_input_attrs_[index].dims[1] * m_input_attrs_[index].dims[3]);
} else {
int height = m_input_attrs_[index].dims[1];
int channel = m_input_attrs_[index].dims[3];
// copy from src to dst with stride
uint8_t *src_ptr = data;
uint8_t *dst_ptr = (uint8_t *)m_input_mems_[index]->virt_addr;
// width-channel elements
int src_wc_elems = width * channel;
int dst_wc_elems = stride * channel;
for (int h = 0; h < height; ++h) {
memcpy(dst_ptr, src_ptr, src_wc_elems);
src_ptr += src_wc_elems;
dst_ptr += dst_wc_elems;
}
}
// Set input tensor memory
auto ret = rknn_set_io_mem(m_rk_ctx_, m_input_mems_[index], &m_input_attrs_[index]);
if (ret < 0) {
INSPIRE_LOGE("rknn_set_io_mem fail! ret = %d", ret);
return -1;
}
} else {
INSPIRE_LOGE("error: assert index < input size");
}
return 0;
}
int32_t RunSession(bool use_raw_output = false) {
// Set output tensor memory
for (uint32_t i = 0; i < m_rk_io_num_.n_output; ++i) {
// set output memory and attribute
auto ret = rknn_set_io_mem(m_rk_ctx_, m_output_mems_[i], &m_output_attrs_[i]);
if (ret < 0) {
INSPIRE_LOGE("rknn_set_io_mem fail! ret = %d", ret);
return -1;
}
}
auto ret = rknn_run(m_rk_ctx_, NULL);
if (ret < 0) {
printf("rknn run error %d\n", ret);
return -1;
}
if (use_raw_output) {
m_output_nchw_.resize(m_rk_io_num_.n_output);
for (uint32_t i = 0; i < m_rk_io_num_.n_output; ++i) {
int num_elements = m_orig_output_attrs_[i].size_with_stride;
m_output_nchw_[i].resize(num_elements);
}
for (uint32_t i = 0; i < m_rk_io_num_.n_output; i++) {
if (m_output_attrs_[i].fmt == RKNN_TENSOR_NC1HWC2) {
int channel = m_orig_output_attrs_[i].dims[1];
int h = m_orig_output_attrs_[i].n_dims > 2 ? m_orig_output_attrs_[i].dims[2] : 1;
int w = m_orig_output_attrs_[i].n_dims > 3 ? m_orig_output_attrs_[i].dims[3] : 1;
int zp = m_output_attrs_[i].zp;
float scale = m_output_attrs_[i].scale;
NC1HWC2_int8_to_NCHW_float((int8_t *)m_output_mems_[i]->virt_addr, m_output_nchw_[i].data(), (int *)m_output_attrs_[i].dims,
channel, h, w, zp, scale);
} else {
int8_t *src = (int8_t *)m_output_mems_[i]->virt_addr;
float *dst = m_output_nchw_[i].data();
for (int index = 0; index < m_output_attrs_[i].n_elems; index++) {
dst[index] = (src[index] - m_output_attrs_[i].zp) * m_output_attrs_[i].scale;
}
}
}
}
return 0;
}
std::vector<float> &GetOutputData(size_t index) {
return m_output_nchw_[index];
}
rknn_tensor_mem *GetOutputRawData(size_t index) {
return m_output_mems_[index];
}
std::vector<rknn_tensor_attr> &GetOutputAttrs() {
return m_output_attrs_;
}
const float *GetOutputDataPtr(const int index) {
return (float *)(m_output_nchw_[index].data());
}
std::vector<unsigned long> GetOutputTensorSize(const int &index) {
std::vector<unsigned long> dims(m_output_attrs_[index].dims, m_output_attrs_[index].dims + m_output_attrs_[index].n_dims);
return dims;
}
~RKNNAdapterNano() {
Release();
}
void Release() {
if (run_) {
for (uint32_t i = 0; i < m_rk_io_num_.n_input; ++i) {
rknn_destroy_mem(m_rk_ctx_, m_input_mems_[i]);
}
for (uint32_t i = 0; i < m_rk_io_num_.n_output; ++i) {
rknn_destroy_mem(m_rk_ctx_, m_output_mems_[i]);
}
if (m_rk_ctx_) {
rknn_destroy(m_rk_ctx_);
}
}
run_ = false;
}
private:
rknn_context m_rk_ctx_;
rknn_input_output_num m_rk_io_num_;
std::vector<rknn_tensor_attr> m_input_attrs_;
std::vector<rknn_tensor_attr> m_output_attrs_;
std::vector<rknn_tensor_attr> m_orig_output_attrs_;
std::vector<rknn_tensor_mem *> m_input_mems_;
std::vector<rknn_tensor_mem *> m_output_mems_;
std::vector<std::vector<float>> m_output_nchw_;
bool run_;
};
#endif // SLEEPMONITORING_RKNN_ADAPTER_NANO_H

191
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper.cpp Normal file
View File

@@ -0,0 +1,191 @@
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include "inference_wrapper_log.h"
#include "inference_wrapper.h"
#ifdef INFERENCE_WRAPPER_ENABLE_MNN
#include "inference_wrapper_mnn.h"
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
// #include "inference_wrapper_rknn.h"
#include "inference_wrapper_rknn_adapter.h"
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN2
#include "inference_wrapper_rknn_adapter_nano.h"
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_COREML
#include "inference_wrapper_coreml.h"
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_TENSORRT
#include "inference_wrapper_tensorrt.h"
#endif
#define TAG "InferenceWrapper"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceWrapper* InferenceWrapper::Create(const InferenceWrapper::EngineType helper_type) {
InferenceWrapper* p = nullptr;
switch (helper_type) {
#ifdef INFERENCE_WRAPPER_ENABLE_MNN
case INFER_MNN:
// PRINT("Use General Inference\n");
p = new InferenceWrapperMNN();
break;
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
case INFER_RKNN:
// PRINT("Use Rknn\n")
// p = new InferenceWrapperRKNN();
p = new InferenceWrapperRKNNAdapter();
break;
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN2
case INFER_RKNN:
// PRINT("Use Rknn2\n");
p = new InferenceWrapperRKNNAdapter();
break;
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_COREML
case INFER_COREML:
// PRINT("Use CoreML\n");
p = new InferenceWrapperCoreML();
break;
#endif
#ifdef INFERENCE_WRAPPER_ENABLE_TENSORRT
case INFER_TENSORRT:
// PRINT("Use TensorRT\n");
p = new InferenceWrapperTensorRT();
break;
#endif
default:
PRINT_E("Unsupported inference helper type (%d)\n", helper_type)
break;
}
if (p == nullptr) {
PRINT_E("Failed to create inference helper\n")
} else {
p->helper_type_ = helper_type;
}
return p;
}
void InferenceWrapper::ConvertNormalizeParameters(InputTensorInfo& tensor_info) {
if (tensor_info.data_type != InputTensorInfo::DataTypeImage)
return;
/* Convert to speeden up normalization: ((src / 255) - mean) / norm = (src - (mean * 255)) * (1 / (255 * norm)) */
for (int32_t i = 0; i < 3; i++) {
tensor_info.normalize.mean[i] *= 255.0f;
tensor_info.normalize.norm[i] *= 255.0f;
tensor_info.normalize.norm[i] = 1.0f / tensor_info.normalize.norm[i];
}
}
void InferenceWrapper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst) {
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] =
(src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
}
}
} else {
/* convert NHWC to NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = (src[i * img_channel + c] - input_tensor_info.normalize.mean[c]) * input_tensor_info.normalize.norm[c];
}
}
}
}
void InferenceWrapper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst) {
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c];
}
}
} else {
/* convert NHWC to NHWC */
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
}
}
void InferenceWrapper::PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst) {
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
uint8_t* src = (uint8_t*)(input_tensor_info.data);
if (input_tensor_info.is_nchw == true) {
/* convert NHWC to NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t c = 0; c < img_channel; c++) {
for (int32_t i = 0; i < img_width * img_height; i++) {
dst[c * img_width * img_height + i] = src[i * img_channel + c] - 128;
}
}
} else {
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[i * img_channel + c] - 128;
}
}
}
}
template <typename T>
void InferenceWrapper::PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T* dst) {
const int32_t img_width = input_tensor_info.GetWidth();
const int32_t img_height = input_tensor_info.GetHeight();
const int32_t img_channel = input_tensor_info.GetChannel();
T* src = static_cast<T*>(input_tensor_info.data);
if ((input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNchw && input_tensor_info.is_nchw) ||
(input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc && !input_tensor_info.is_nchw)) {
std::copy(src, src + input_tensor_info.GetElementNum(), dst);
} else if (input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNchw) {
/* NCHW -> NHWC */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[i * img_channel + c] = src[c * (img_width * img_height) + i];
}
}
} else if (input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) {
/* NHWC -> NCHW */
#pragma omp parallel for num_threads(num_thread)
for (int32_t i = 0; i < img_width * img_height; i++) {
for (int32_t c = 0; c < img_channel; c++) {
dst[c * (img_width * img_height) + i] = src[i * img_channel + c];
}
}
}
}
template void InferenceWrapper::PreProcessBlob<float>(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
template void InferenceWrapper::PreProcessBlob<int32_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int32_t* dst);
template void InferenceWrapper::PreProcessBlob<int64_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int64_t* dst);
template void InferenceWrapper::PreProcessBlob<uint8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
template void InferenceWrapper::PreProcessBlob<int8_t>(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);

257
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper.h Normal file
View File

@@ -0,0 +1,257 @@
#ifndef INFERENCE_WRAPPER_
#define INFERENCE_WRAPPER_
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
class TensorInfo {
public:
enum {
TensorTypeNone,
TensorTypeUint8,
TensorTypeInt8,
TensorTypeFp32,
TensorTypeInt32,
TensorTypeInt64,
};
public:
TensorInfo() : name(""), id(-1), tensor_type(TensorTypeNone), is_nchw(true) {}
~TensorInfo() {}
int32_t GetElementNum() const {
int32_t element_num = 1;
for (const auto& dim : tensor_dims) {
element_num *= dim;
}
return element_num;
}
int32_t GetBatch() const {
if (tensor_dims.size() <= 0)
return -1;
return tensor_dims[0];
}
int32_t GetChannel() const {
if (is_nchw) {
if (tensor_dims.size() <= 1)
return -1;
return tensor_dims[1];
} else {
if (tensor_dims.size() <= 3)
return -1;
return tensor_dims[3];
}
}
int32_t GetHeight() const {
if (is_nchw) {
if (tensor_dims.size() <= 2)
return -1;
return tensor_dims[2];
} else {
if (tensor_dims.size() <= 1)
return -1;
return tensor_dims[1];
}
}
int32_t GetWidth() const {
if (is_nchw) {
if (tensor_dims.size() <= 3)
return -1;
return tensor_dims[3];
} else {
if (tensor_dims.size() <= 2)
return -1;
return tensor_dims[2];
}
}
public:
std::string name;
int32_t id;
int32_t tensor_type;
std::vector<int32_t> tensor_dims;
bool is_nchw;
};
class InputTensorInfo : public TensorInfo {
public:
enum {
DataTypeImage,
DataTypeBlobNhwc, // data_ which already finished preprocess(color conversion, resize, normalize_, etc.)
DataTypeBlobNchw,
};
public:
InputTensorInfo()
: data(nullptr),
data_type(DataTypeImage),
image_info({-1, -1, -1, -1, -1, -1, -1, true, false}),
normalize({0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f}) {}
InputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true) : InputTensorInfo() {
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~InputTensorInfo() {}
public:
void* data;
int32_t data_type;
struct {
int32_t width;
int32_t height;
int32_t channel;
int32_t crop_x;
int32_t crop_y;
int32_t crop_width;
int32_t crop_height;
bool is_bgr; // used when channel == 3 (true: BGR, false: RGB)
bool swap_color;
} image_info;
struct {
float mean[3];
float norm[3];
} normalize;
};
class OutputTensorInfo : public TensorInfo {
public:
OutputTensorInfo() : data(nullptr), quant({1.0f, 0}), data_fp32_(nullptr) {}
OutputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true) : OutputTensorInfo() {
name = name_;
tensor_type = tensor_type_;
is_nchw = is_nchw_;
}
~OutputTensorInfo() {
if (data_fp32_ != nullptr) {
delete[] data_fp32_;
}
}
float* GetDataAsFloat() {
if (tensor_type == TensorTypeUint8 || tensor_type == TensorTypeInt8) {
if (data_fp32_ == nullptr) {
data_fp32_ = new float[GetElementNum()];
}
if (tensor_type == TensorTypeUint8) {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const uint8_t* val_uint8 = static_cast<const uint8_t*>(data);
float val_float = (val_uint8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
} else {
#pragma omp parallel
for (int32_t i = 0; i < GetElementNum(); i++) {
const int8_t* val_int8 = static_cast<const int8_t*>(data);
float val_float = (val_int8[i] - quant.zero_point) * quant.scale;
data_fp32_[i] = val_float;
}
}
return data_fp32_;
} else if (tensor_type == TensorTypeFp32) {
return static_cast<float*>(data);
} else {
return nullptr;
}
}
public:
void* data;
struct {
float scale;
int32_t zero_point;
} quant;
private:
float* data_fp32_;
};
namespace cv {
class Mat;
};
class InferenceWrapper {
public:
enum {
WrapperOk = 0,
WrapperError = -1,
};
typedef enum {
DEFAULT_CPU,
MMM_CUDA,
COREML_CPU,
COREML_GPU,
COREML_ANE,
TENSORRT_CUDA,
} SpecialBackend;
typedef enum {
INFER_MNN,
INFER_RKNN,
INFER_COREML,
INFER_TENSORRT,
} EngineType;
public:
static InferenceWrapper* Create(const EngineType helper_type);
public:
virtual ~InferenceWrapper() {}
virtual int32_t SetNumThreads(const int32_t num_threads) = 0;
virtual int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t Finalize(void) = 0;
virtual int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
virtual int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) = 0;
virtual int32_t SetSpecialBackend(SpecialBackend backend) {
special_backend_ = backend;
return WrapperOk;
};
virtual int32_t SetDevice(int32_t device_id) {
device_id_ = device_id;
return WrapperOk;
};
virtual int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) = 0;
virtual std::vector<std::string> GetInputNames() = 0;
protected:
void ConvertNormalizeParameters(InputTensorInfo& tensor_info);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst);
void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst);
template <typename T>
void PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T* dst);
protected:
EngineType helper_type_;
SpecialBackend special_backend_ = DEFAULT_CPU;
int32_t device_id_ = 0;
};
#endif

215
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_coreml.cpp Normal file
View File

@@ -0,0 +1,215 @@
#ifdef INFERENCE_WRAPPER_ENABLE_COREML
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include <MNN/ImageProcess.hpp>
#include "inference_wrapper_log.h"
#include "inference_wrapper_coreml.h"
#include "log.h"
#define TAG "InferenceWrapperCoreML"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
using namespace inspire;
InferenceWrapperCoreML::InferenceWrapperCoreML() {
num_threads_ = 1;
}
InferenceWrapperCoreML::~InferenceWrapperCoreML() {}
int32_t InferenceWrapperCoreML::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return WrapperOk;
}
int32_t InferenceWrapperCoreML::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
return WrapperOk;
}
int32_t InferenceWrapperCoreML::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
PRINT_E("CoreML does not yet support buffer initialization of the model\n");
return WrapperError;
}
int32_t InferenceWrapperCoreML::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
// LOG_INFO("init MNN");
/*** Create network ***/
net_.reset(new CoreMLAdapter());
auto ret = net_->readFromFile(model_filename);
if (ret != WrapperOk) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
return WrapperError;
}
if (special_backend_ == COREML_CPU || special_backend_ == DEFAULT_CPU) {
net_->setInferenceMode(CoreMLAdapter::InferenceMode::CPU);
} else if (special_backend_ == COREML_GPU) {
net_->setInferenceMode(CoreMLAdapter::InferenceMode::GPU);
} else if (special_backend_ == COREML_ANE) {
net_->setInferenceMode(CoreMLAdapter::InferenceMode::ANE);
} else {
PRINT_E("Unsupported backend (%d)\n, defaulting to ANE", special_backend_);
net_->setInferenceMode(CoreMLAdapter::InferenceMode::ANE);
}
if (!net_) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
};
int32_t InferenceWrapperCoreML::Finalize(void) {
net_.reset();
return WrapperOk;
}
int32_t InferenceWrapperCoreML::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) {
// Currently only single-input models are supported
for (const auto& input_tensor_info : input_tensor_info_list) {
input_tensor_.reset(MNN::Tensor::create<float>(
std::vector<int>{1, 3, input_tensor_info.image_info.height, input_tensor_info.image_info.width}, nullptr, MNN::Tensor::CAFFE));
if (input_tensor_ == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
INSPIRE_LOGE("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return WrapperError;
}
if (input_tensor_info.data_type == InputTensorInfo::DataTypeImage) {
/* Crop */
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) ||
(input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
PRINT_E("Crop is not supported\n");
return WrapperError;
}
MNN::CV::ImageProcess::Config image_processconfig;
/* Convert color type */
// LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
// LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
// !!!!!! BUG !!!!!!!!!
// When initializing, setting the image channel to 3 and the tensor channel to 1,
// and configuring the processing to convert the color image to grayscale may cause some bugs.
// For example, the image channel might automatically change to 1.
// This issue has not been fully investigated,
// so it's necessary to manually convert the image to grayscale before input.
// !!!!!! BUG !!!!!!!!!
if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
if (input_tensor_info.image_info.swap_color) {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::RGB : MNN::CV::BGR;
} else {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
}
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::GRAY;
} else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
image_processconfig.destFormat = MNN::CV::GRAY;
// LOGD("2gray");
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::BGR;
} else {
PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel());
return WrapperError;
}
/* Normalize image */
std::memcpy(image_processconfig.mean, input_tensor_info.normalize.mean, sizeof(image_processconfig.mean));
std::memcpy(image_processconfig.normal, input_tensor_info.normalize.norm, sizeof(image_processconfig.normal));
/* Resize image */
image_processconfig.filterType = MNN::CV::BILINEAR;
MNN::CV::Matrix trans;
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(),
static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
/* Do pre-process */
std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(image_processconfig));
pretreat->setMatrix(trans);
// LOGD("k1");
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width,
input_tensor_info.image_info.crop_height, 0, input_tensor_.get());
} else if ((input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) ||
(input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNchw)) {
std::unique_ptr<MNN::Tensor> tensor;
if (input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) {
tensor.reset(new MNN::Tensor(input_tensor_.get(), MNN::Tensor::TENSORFLOW));
} else {
tensor.reset(new MNN::Tensor(input_tensor_.get(), MNN::Tensor::CAFFE));
}
if (tensor->getType().code == halide_type_float) {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<float>()[i] = static_cast<float*>(input_tensor_info.data)[i];
}
} else {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<uint8_t>()[i] = static_cast<uint8_t*>(input_tensor_info.data)[i];
}
}
input_tensor_->copyFromHostTensor(tensor.get());
} else {
PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type);
return WrapperError;
}
auto p = input_tensor_->host<float>();
net_->setInput(input_tensor_info.name.c_str(), reinterpret_cast<const char*>(p));
}
return WrapperOk;
}
int32_t InferenceWrapperCoreML::Process(std::vector<OutputTensorInfo>& output_tensor_info_list) {
auto ret = net_->forward();
if (ret != COREML_HSUCCEED) {
PRINT_E("Failed to forward\n");
return WrapperError;
}
// out_mat_list_.clear();
for (auto& output_tensor_info : output_tensor_info_list) {
// auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
const char* output_tensor = net_->getOutput(output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return WrapperError;
}
output_tensor_info.data = (void*)output_tensor;
output_tensor_info.tensor_dims.clear();
const std::vector<int>& output_shape = net_->getOutputShapeByName(output_tensor_info.name);
int size = 1;
for (int32_t dim = 0; dim < output_shape.size(); dim++) {
output_tensor_info.tensor_dims.push_back(output_shape[dim]);
size *= output_shape[dim];
}
}
return WrapperOk;
}
std::vector<std::string> InferenceWrapperCoreML::GetInputNames() {
return input_names_;
}
int32_t InferenceWrapperCoreML::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
PRINT_E("Currently, CoreML does not support input resizing\n");
return 0;
}
#endif // INFERENCE_WRAPPER_ENABLE_COREML

43
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_coreml.h Normal file
View File

@@ -0,0 +1,43 @@
#ifndef INFERENCE_WRAPPER_COREML_
#define INFERENCE_WRAPPER_COREML_
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
#include "coreml/CoreMLAdapter.h"
#include <MNN/ImageProcess.hpp>
#include "inference_wrapper.h"
class InferenceWrapperCoreML : public InferenceWrapper {
public:
InferenceWrapperCoreML();
~InferenceWrapperCoreML() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
private:
std::unique_ptr<CoreMLAdapter> net_;
int32_t num_threads_;
std::vector<std::string> input_names_;
/** Using MNN imageprocess to do Image Preprocessing */
std::unique_ptr<MNN::Tensor> input_tensor_;
};
#endif // INFERENCE_WRAPPER_COREML_

31
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_log.h Normal file
View File

@@ -0,0 +1,31 @@
#ifndef INFERENCE_WRAPPER_LOG_
#define INFERENCE_WRAPPER_LOG_
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#if defined(ANDROID) || defined(__ANDROID__)
#define CV_COLOR_IS_RGB
#include <android/log.h>
#define INFERENCE_WRAPPER_LOG_NDK_TAG "Inspireface-Native-Inference"
#define INFERENCE_WRAPPER_LOG_PRINT_(...) __android_log_print(ANDROID_LOG_INFO, INFERENCE_WRAPPER_LOG_NDK_TAG, __VA_ARGS__)
#else
#define INFERENCE_WRAPPER_LOG_PRINT_(...) printf(__VA_ARGS__)
#endif
#define INFERENCE_WRAPPER_LOG_PRINT(INFERENCE_WRAPPER_LOG_PRINT_TAG, ...) \
do { \
INFERENCE_WRAPPER_LOG_PRINT_("[" INFERENCE_WRAPPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_WRAPPER_LOG_PRINT_(__VA_ARGS__); \
} while (0);
#define INFERENCE_WRAPPER_LOG_PRINT_E(INFERENCE_WRAPPER_LOG_PRINT_TAG, ...) \
do { \
INFERENCE_WRAPPER_LOG_PRINT_("[ERR: " INFERENCE_WRAPPER_LOG_PRINT_TAG "][%d] ", __LINE__); \
INFERENCE_WRAPPER_LOG_PRINT_(__VA_ARGS__); \
} while (0);
#endif

210
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_mnn.cpp → cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_mnn.cpp
View File

@@ -1,19 +1,3 @@
/* Copyright 2021 iwatake2222
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/*** Include ***/
/* for general */
#include <cstdint>
#include <cstdlib>
#include <cmath>
@@ -23,74 +7,61 @@ limitations under the License.
#include <array>
#include <algorithm>
#include <chrono>
/* for MNN */
#include <MNN/ImageProcess.hpp>
#include <MNN/Interpreter.hpp>
#include <MNN/AutoTime.hpp>
/* for My modules */
#include "inference_helper_log.h"
#include "inference_helper_mnn.h"
#include "inference_wrapper_log.h"
#include "inference_wrapper_mnn.h"
#include "log.h"
/*** Macro ***/
#define TAG "InferenceHelperMnn"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
#define TAG "InferenceWrapperMNN"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
using namespace inspire;
/*** Function ***/
InferenceHelperMnn::InferenceHelperMnn()
{
InferenceWrapperMNN::InferenceWrapperMNN() {
num_threads_ = 1;
}
InferenceHelperMnn::~InferenceHelperMnn()
{
}
InferenceWrapperMNN::~InferenceWrapperMNN() {}
int32_t InferenceHelperMnn::SetNumThreads(const int32_t num_threads)
{
int32_t InferenceWrapperMNN::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperMnn::SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops)
{
PRINT("[WARNING] This method is not supported\n");
return kRetOk;
}
int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
int32_t InferenceWrapperMNN::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
/* Check tensor info fits the info from model */
for (auto& input_tensor_info : input_tensor_info_list) {
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
if (input_tensor == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
// LOGD("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return kRetErr;
// LOGD("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return WrapperError;
}
if ((input_tensor->getType().code == halide_type_float) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32)) {
if ((input_tensor->getType().code == halide_type_float) && (input_tensor_info.tensor_type == TensorInfo::TensorTypeFp32)) {
/* OK */
} else if ((input_tensor->getType().code == halide_type_uint) && (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8)) {
} else if ((input_tensor->getType().code == halide_type_uint) && (input_tensor_info.tensor_type == TensorInfo::TensorTypeUint8)) {
/* OK */
} else {
PRINT_E("Incorrect input tensor type (%d, %d)\n", input_tensor->getType().code, input_tensor_info.tensor_type);
return kRetErr;
return WrapperError;
}
if ((input_tensor->channel() != -1) && (input_tensor->height() != -1) && (input_tensor->width() != -1)) {
if (input_tensor_info.GetChannel() != -1) {
if ((input_tensor->channel() == input_tensor_info.GetChannel()) && (input_tensor->height() == input_tensor_info.GetHeight()) && (input_tensor->width() == input_tensor_info.GetWidth())) {
if ((input_tensor->channel() == input_tensor_info.GetChannel()) && (input_tensor->height() == input_tensor_info.GetHeight()) &&
(input_tensor->width() == input_tensor_info.GetWidth())) {
/* OK */
} else {
INSPIRE_LOGW("W: %d != %d", input_tensor->width() , input_tensor_info.GetWidth());
INSPIRE_LOGW("H: %d != %d", input_tensor->height() , input_tensor_info.GetHeight());
INSPIRE_LOGW("C: %d != %d", input_tensor->channel() , input_tensor_info.GetChannel());
INSPIRE_LOGW("W: %d != %d", input_tensor->width(), input_tensor_info.GetWidth());
INSPIRE_LOGW("H: %d != %d", input_tensor->height(), input_tensor_info.GetHeight());
INSPIRE_LOGW("C: %d != %d", input_tensor->channel(), input_tensor_info.GetChannel());
INSPIRE_LOGW("There may be some risk of input that is not used by model default");
net_->resizeTensor(input_tensor, { 1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth() });
net_->resizeTensor(input_tensor,
{1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth()});
net_->resizeSession(session_);
return kRetOk;
return WrapperOk;
}
} else {
PRINT("Input tensor size is set from the model\n");
@@ -103,12 +74,12 @@ int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>
if (input_tensor_info.GetChannel() != -1) {
PRINT("Input tensor size is resized\n");
/* In case the input size is not fixed */
net_->resizeTensor(input_tensor, { 1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth() });
net_->resizeTensor(input_tensor, {1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth()});
net_->resizeSession(session_);
INSPIRE_LOGE("GO RESIZE");
} else {
PRINT_E("Model input size is not set\n");
return kRetErr;
return WrapperError;
}
}
}
@@ -116,7 +87,7 @@ int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return kRetErr;
return WrapperError;
}
/* Output size is set when run inference later */
}
@@ -126,45 +97,32 @@ int32_t InferenceHelperMnn::ParameterInitialization(std::vector<InputTensorInfo>
ConvertNormalizeParameters(input_tensor_info);
}
/* Check if tensor info is set */
for (const auto& input_tensor_info : input_tensor_info_list) {
for (const auto& dim : input_tensor_info.tensor_dims) {
if (dim <= 0) {
PRINT_E("Invalid tensor size\n");
return kRetErr;
return WrapperError;
}
}
}
//for (const auto& output_tensor_info : output_tensor_info_list) {
// for (const auto& dim : output_tensor_info.tensor_dims) {
// if (dim <= 0) {
// PRINT_E("Invalid tensor size\n");
// return kRetErr;
// }
// }
//}
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperMnn::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
// PRINT("-Initialize\n");
// LOGD("-Initialize");
/*** Create network ***/
// LOG_INFO("init MNN");
// PRINT_E("init MNN");
int32_t InferenceWrapperMNN::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
net_.reset(MNN::Interpreter::createFromBuffer(model_buffer, model_size));
if (!net_) {
PRINT_E("Failed to load model model buffer\n");
return kRetErr;
return WrapperError;
}
MNN::ScheduleConfig scheduleConfig;
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
MNN::BackendConfig bnconfig;
bnconfig.power = MNN::BackendConfig::Power_High;
bnconfig.precision = MNN::BackendConfig::Precision_High;
if (special_backend_ == kMnnCuda) {
bnconfig.precision = MNN::BackendConfig::Precision_Normal;
if (special_backend_ == MMM_CUDA) {
INSPIRE_LOGD("Enable CUDA");
scheduleConfig.type = MNN_FORWARD_CUDA;
bnconfig.power = MNN::BackendConfig::Power_Normal;
@@ -175,33 +133,28 @@ int32_t InferenceHelperMnn::Initialize(char* model_buffer, int model_size, std::
scheduleConfig.backendConfig = &bnconfig;
session_ = net_->createSession(scheduleConfig);
// LOG_INFO("-INPUT: {}", net_->getSessionInputAll(session_).size());
// PRINT("-INPUT: %lu", net_->getSessionInputAll(session_).size());
// LOGD("-INPUT: %lu", net_->getSessionInputAll(session_).size());
for (auto &item: net_->getSessionInputAll(session_)) {
for (auto& item : net_->getSessionInputAll(session_)) {
input_names_.push_back(item.first.c_str());
}
if (!session_) {
PRINT_E("Failed to create session\n");
return kRetErr;
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceHelperMnn::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list)
{
// LOG_INFO("init MNN");
/*** Create network ***/
int32_t InferenceWrapperMNN::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
net_.reset(MNN::Interpreter::createFromFile(model_filename.c_str()));
if (!net_) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
return kRetErr;
return WrapperError;
}
MNN::ScheduleConfig scheduleConfig;
scheduleConfig.type = MNN_FORWARD_CPU;
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
scheduleConfig.numThread = num_threads_; // it seems, setting 1 has better performance on Android
// MNN::BackendConfig bnconfig;
// bnconfig.power = MNN::BackendConfig::Power_High;
// bnconfig.precision = MNN::BackendConfig::Precision_Low;
@@ -209,52 +162,46 @@ int32_t InferenceHelperMnn::Initialize(const std::string& model_filename, std::v
session_ = net_->createSession(scheduleConfig);
if (!session_) {
PRINT_E("Failed to create session\n");
return kRetErr;
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
};
int32_t InferenceHelperMnn::Finalize(void)
{
int32_t InferenceWrapperMNN::Finalize(void) {
net_->releaseSession(session_);
net_->releaseModel();
net_.reset();
out_mat_list_.clear();
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list)
{
// for (auto &item: net_->getSessionInputAll(session_)) {
// PRINT("sss: %s", item.first.c_str());
// }
int32_t InferenceWrapperMNN::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) {
for (const auto& input_tensor_info : input_tensor_info_list) {
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
if (input_tensor == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
INSPIRE_LOGE("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return kRetErr;
return WrapperError;
}
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) {
if (input_tensor_info.data_type == InputTensorInfo::DataTypeImage) {
/* Crop */
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) ||
(input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
PRINT_E("Crop is not supported\n");
return kRetErr;
return WrapperError;
}
MNN::CV::ImageProcess::Config image_processconfig;
/* Convert color type */
// LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
// LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
// LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
// LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
// !!!!!! BUG !!!!!!!!!
// When initializing, setting the image channel to 3 and the tensor channel to 1,
// and configuring the processing to convert the color image to grayscale may cause some bugs.
// For example, the image channel might automatically change to 1.
// This issue has not been fully investigated,
// When initializing, setting the image channel to 3 and the tensor channel to 1,
// and configuring the processing to convert the color image to grayscale may cause some bugs.
// For example, the image channel might automatically change to 1.
// This issue has not been fully investigated,
// so it's necessary to manually convert the image to grayscale before input.
// !!!!!! BUG !!!!!!!!!
@@ -271,33 +218,35 @@ int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input
} else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
image_processconfig.destFormat = MNN::CV::GRAY;
// LOGD("2gray");
// LOGD("2gray");
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::BGR;
} else {
PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel());
return kRetErr;
return WrapperError;
}
/* Normalize image */
std::memcpy(image_processconfig.mean, input_tensor_info.normalize.mean, sizeof(image_processconfig.mean));
std::memcpy(image_processconfig.normal, input_tensor_info.normalize.norm, sizeof(image_processconfig.normal));
/* Resize image */
image_processconfig.filterType = MNN::CV::BILINEAR;
MNN::CV::Matrix trans;
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(), static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(),
static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
/* Do pre-process */
std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(image_processconfig));
pretreat->setMatrix(trans);
// LOGD("k1");
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width, input_tensor_info.image_info.crop_height, 0, input_tensor);
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width,
input_tensor_info.image_info.crop_height, 0, input_tensor);
} else if ( (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) ) {
} else if ((input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) ||
(input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNchw)) {
std::unique_ptr<MNN::Tensor> tensor;
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) {
if (input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) {
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::TENSORFLOW));
} else {
tensor.reset(new MNN::Tensor(input_tensor, MNN::Tensor::CAFFE));
@@ -314,14 +263,13 @@ int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input
input_tensor->copyFromHostTensor(tensor.get());
} else {
PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type);
return kRetErr;
return WrapperError;
}
}
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperMnn::Process(std::vector<OutputTensorInfo>& output_tensor_info_list)
{
int32_t InferenceWrapperMNN::Process(std::vector<OutputTensorInfo>& output_tensor_info_list) {
net_->runSession(session_);
out_mat_list_.clear();
@@ -329,7 +277,7 @@ int32_t InferenceHelperMnn::Process(std::vector<OutputTensorInfo>& output_tensor
auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return kRetErr;
return WrapperError;
}
auto dimType = output_tensor->getDimensionType();
@@ -337,35 +285,35 @@ int32_t InferenceHelperMnn::Process(std::vector<OutputTensorInfo>& output_tensor
output_tensor->copyToHostTensor(outputUser.get());
auto type = outputUser->getType();
if (type.code == halide_type_float) {
output_tensor_info.tensor_type = TensorInfo::kTensorTypeFp32;
output_tensor_info.tensor_type = TensorInfo::TensorTypeFp32;
output_tensor_info.data = outputUser->host<float>();
} else if (type.code == halide_type_uint && type.bytes() == 1) {
output_tensor_info.tensor_type = TensorInfo::kTensorTypeUint8;
output_tensor_info.tensor_type = TensorInfo::TensorTypeUint8;
output_tensor_info.data = outputUser->host<uint8_t>();
} else {
PRINT_E("Unexpected data type\n");
return kRetErr;
return WrapperError;
}
output_tensor_info.tensor_dims.clear();
for (int32_t dim = 0; dim < outputUser->dimensions(); dim++) {
output_tensor_info.tensor_dims.push_back(outputUser->length(dim));
}
out_mat_list_.push_back(std::move(outputUser)); // store data in member variable so that data keep exist
out_mat_list_.push_back(std::move(outputUser)); // store data in member variable so that data keep exist
}
return kRetOk;
return WrapperOk;
}
std::vector<std::string> InferenceHelperMnn::GetInputNames() {
std::vector<std::string> InferenceWrapperMNN::GetInputNames() {
return input_names_;
}
int32_t InferenceHelperMnn::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
int32_t InferenceWrapperMNN::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
for (const auto& input_tensor_info : input_tensor_info_list) {
auto input_tensor = net_->getSessionInput(session_, input_tensor_info.name.c_str());
net_->resizeTensor(input_tensor, { 1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth() });
net_->resizeTensor(input_tensor, {1, input_tensor_info.GetChannel(), input_tensor_info.GetHeight(), input_tensor_info.GetWidth()});
net_->resizeSession(session_);
}
return 0;

43
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_mnn.h Normal file
View File

@@ -0,0 +1,43 @@
#ifndef INFERENCE_WRAPPER_MNN_
#define INFERENCE_WRAPPER_MNN_
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
#include <MNN/ImageProcess.hpp>
#include <MNN/Interpreter.hpp>
#include <MNN/AutoTime.hpp>
#include "inference_wrapper.h"
class InferenceWrapperMNN : public InferenceWrapper {
public:
InferenceWrapperMNN();
~InferenceWrapperMNN() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
private:
std::unique_ptr<MNN::Interpreter> net_;
MNN::Session* session_;
std::vector<std::unique_ptr<MNN::Tensor>> out_mat_list_;
int32_t num_threads_;
std::vector<std::string> input_names_;
};
#endif

154
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_rknn.cpp → cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn.cpp
View File

@@ -1,11 +1,8 @@
//
// Created by tunm on 2023/2/5.
//
/*** Include ***/
/* for general */
#ifdef INFERENCE_HELPER_ENABLE_RKNN
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
#include <iostream>
#include <cstdint>
#include <cstdlib>
@@ -16,15 +13,13 @@
#include <array>
#include <algorithm>
#include <chrono>
#include "inference_helper_rknn.h"
#include "inference_helper_log.h"
#include "inference_wrapper_rknn.h"
#include "inference_wrapper_log.h"
#include "log.h"
static unsigned char* load_data(FILE* fp, size_t ofst, size_t sz)
{
static unsigned char* load_data(FILE* fp, size_t ofst, size_t sz) {
unsigned char* data;
int ret;
int ret;
data = NULL;
@@ -47,8 +42,7 @@ static unsigned char* load_data(FILE* fp, size_t ofst, size_t sz)
return data;
}
static unsigned char* load_model(const char* filename, int* model_size)
{
static unsigned char* load_model(const char* filename, int* model_size) {
FILE* fp;
unsigned char* data;
@@ -69,127 +63,114 @@ static unsigned char* load_model(const char* filename, int* model_size)
return data;
}
/*** Macro ***/
#define TAG "InferenceHelperRknn"
#define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__)
#define TAG "InferenceWrapperRKNN"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceHelperRKNN::InferenceHelperRKNN() {
InferenceWrapperRKNN::InferenceWrapperRKNN() {
num_threads_ = 1;
}
InferenceHelperRKNN::~InferenceHelperRKNN() {
}
InferenceWrapperRKNN::~InferenceWrapperRKNN() {}
int32_t InferenceHelperRKNN::SetNumThreads(const int32_t num_threads) {
int32_t InferenceWrapperRKNN::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperRKNN::SetCustomOps(const std::vector<std::pair<const char *, const void *>> &custom_ops) {
PRINT("[WARNING] This method is not supported\n");
return kRetOk;
}
int32_t InferenceHelperRKNN::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
auto ret = rknn_query(net_, RKNN_QUERY_IN_OUT_NUM, &rk_io_num_,
sizeof(rk_io_num_));
int32_t InferenceWrapperRKNN::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
auto ret = rknn_query(net_, RKNN_QUERY_IN_OUT_NUM, &rk_io_num_, sizeof(rk_io_num_));
if (ret != RKNN_SUCC) {
PRINT_E("rknn_query ctx fail! ret=%d\n", ret)
return kRetErr;
return WrapperError;
}
// for (size_t index = 0; index < input_tensor_info_list.size(); ++index) {
// auto &input_tensor_info = input_tensor_info_list[index];
// }
std::vector<rknn_tensor_attr> output_attrs_;
output_attrs_.resize(rk_io_num_.n_output);
for (int i = 0; i < rk_io_num_.n_output; ++i) {
// memset(&output_attrs_[i], 0, sizeof(output_attrs_[i]));
// memset(&output_tensors_[i], 0, sizeof(output_tensors_[i]));
// memset(&output_attrs_[i], 0, sizeof(output_attrs_[i]));
// memset(&output_tensors_[i], 0, sizeof(output_tensors_[i]));
output_attrs_[i].index = i;
ret = rknn_query(net_, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs_[i]),
sizeof(rknn_tensor_attr));
auto &output = output_tensor_info_list[i];
ret = rknn_query(net_, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs_[i]), sizeof(rknn_tensor_attr));
auto& output = output_tensor_info_list[i];
output.tensor_dims.clear();
for (int j = 0; j < output_attrs_[i].n_dims; ++j) {
output.tensor_dims.push_back(output_attrs_[i].dims[j]);
std::cout << "dim: " << output_attrs_[i].dims[j] << std::endl;
}
// std::cout << output_attrs_[i].n_dims << std::endl;
// std::cout << output_attrs_[i].n_dims << std::endl;
}
return kRetOk;
return WrapperOk;
}
int32_t
InferenceHelperRKNN::Initialize(const std::string &model_filename, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
int32_t InferenceWrapperRKNN::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
int model_data_size = 0;
unsigned char* model_data = load_model(model_filename.c_str(), &model_data_size);
int ret = rknn_init(&net_, model_data, model_data_size, 0);
if (ret < 0) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str())
return kRetErr;
return WrapperError;
}
rknn_sdk_version version;
ret = rknn_query(net_, RKNN_QUERY_SDK_VERSION, &version, sizeof(rknn_sdk_version));
if (ret < 0) {
PRINT_E("rknn_init error ret=%d\n", ret)
return kRetErr;
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceHelperRKNN::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) {
int32_t InferenceWrapperRKNN::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
int ret = rknn_init(&net_, model_buffer, model_size, 0);
if (ret < 0) {
PRINT_E("rknn_init error ret=%d\n", ret)
return kRetErr;
return WrapperError;
}
rknn_sdk_version version;
ret = rknn_query(net_, RKNN_QUERY_SDK_VERSION, &version, sizeof(rknn_sdk_version));
if (ret < 0) {
PRINT_E("rknn_init error ret=%d\n", ret)
return kRetErr;
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceHelperRKNN::Finalize(void) {
int32_t InferenceWrapperRKNN::Finalize(void) {
rknn_destroy(net_);
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperRKNN::PreProcess(const std::vector<InputTensorInfo> &input_tensor_info_list) {
int32_t InferenceWrapperRKNN::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) {
/* Check tensor info fits the info from model */
if (input_tensor_info_list.size() != rk_io_num_.n_input) {
PRINT_E("The inputs quantity is inconsistent: i: %d, m: %d", input_tensor_info_list.size(), rk_io_num_.n_input);
return kRetErr;
return WrapperError;
}
std::vector<rknn_input> input_tensors_;
for (size_t index = 0; index < input_tensor_info_list.size(); index++) {
auto &input_tensor_info = input_tensor_info_list[index];
if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) {
auto& input_tensor_info = input_tensor_info_list[index];
if (input_tensor_info.data_type == InputTensorInfo::DataTypeImage) {
/* Crop */
// Not Implement
/* Convert color type */
// Not Implement
rknn_input input;
if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8) {
if (input_tensor_info.tensor_type == TensorInfo::TensorTypeUint8) {
input.type = RKNN_TENSOR_UINT8;
} else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) {
} else if (input_tensor_info.tensor_type == TensorInfo::TensorTypeFp32) {
input.type = RKNN_TENSOR_FLOAT32;
} else {
PRINT_E("Unsupported input type.")
return kRetErr;
return WrapperError;
}
if (input_tensor_info.is_nchw) {
input.fmt = RKNN_TENSOR_NCHW;
@@ -204,65 +185,62 @@ int32_t InferenceHelperRKNN::PreProcess(const std::vector<InputTensorInfo> &inpu
input_tensors_.push_back(input);
}
}
// INSPIRE_LOGD("Prepare data!");
// INSPIRE_LOGD("Prepare data!");
int ret = rknn_inputs_set(net_, input_tensor_info_list.size(), input_tensors_.data());
// INSPIRE_LOGD("Set data!");
if (ret < 0){
// INSPIRE_LOGD("Set data!");
if (ret < 0) {
PRINT_E("rknn_run fail! ret=%d", ret)
return kRetErr;
return WrapperError;
}
return kRetOk;
return WrapperOk;
}
int32_t InferenceHelperRKNN::Process(std::vector<OutputTensorInfo> &output_tensor_info_list) {
int32_t InferenceWrapperRKNN::Process(std::vector<OutputTensorInfo>& output_tensor_info_list) {
if (output_tensor_info_list.size() != rk_io_num_.n_output) {
PRINT_E("The outputs quantity is inconsistent")
return kRetErr;
return WrapperError;
}
auto ret = rknn_run(net_, NULL);
if (ret < 0){
if (ret < 0) {
PRINT_E("rknn_run fail! ret=%d", ret)
return kRetErr;
return WrapperError;
}
for (size_t index = 0; index < output_tensor_info_list.size(); index++) {
auto &output_tensor = output_tensor_info_list[index];
auto& output_tensor = output_tensor_info_list[index];
rknn_output output;
if (output_tensor.tensor_type == TensorInfo::kTensorTypeFp32) {
if (output_tensor.tensor_type == TensorInfo::TensorTypeFp32) {
output.want_float = 1;
INSPIRE_LOGD("want_float=1");
}
output.is_prealloc = 0;
output_tensors_.push_back(output);
// output.want_float = 1; // float
// output.want_float = 1; // float
}
ret = rknn_outputs_get(net_, output_tensor_info_list.size(), output_tensors_.data(), NULL);
if (ret < 0){
if (ret < 0) {
PRINT_E("rknn_run fail! ret=%d", ret)
return kRetErr;
return WrapperError;
}
for (size_t index = 0; index < output_tensor_info_list.size(); index++) {
auto &output_tensor = output_tensor_info_list[index];
auto &output = output_tensors_[index];
auto& output_tensor = output_tensor_info_list[index];
auto& output = output_tensors_[index];
output_tensor.data = output.buf;
float* outBlob = (float* )output.buf;
// output_tensor.
float* outBlob = (float*)output.buf;
// output_tensor.
}
return kRetOk;
return WrapperOk;
}
std::vector<std::string> InferenceHelperRKNN::GetInputNames() {
std::vector<std::string> InferenceWrapperRKNN::GetInputNames() {
return input_names_;
}
int32_t InferenceHelperRKNN::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
int32_t InferenceWrapperRKNN::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
// The function is not supported
return 0;
}
#endif // INFERENCE_HELPER_ENABLE_RKNN
#endif // INFERENCE_WRAPPER_ENABLE_RKNN

43
cpp-package/inspireface/cpp/inspireface/middleware/inference_helper/inference_helper_rknn.h → cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn.h
View File

@@ -1,39 +1,38 @@
//
// Created by tunm on 2023/2/5.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef RKNPU_PTOCR_INFERENCE_HELPER_RKNN_H
#define RKNPU_PTOCR_INFERENCE_HELPER_RKNN_H
#ifdef INFERENCE_HELPER_ENABLE_RKNN
#ifndef RKNPU_PTOCR_INFERENCE_WRAPPER_RKNN_H
#define RKNPU_PTOCR_INFERENCE_WRAPPER_RKNN_H
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
/* for general */
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
/* for RKNN */
#include <rknn_api.h>
#include "inference_wrapper.h"
/* for My modules */
#include "inference_helper.h"
class InferenceHelperRKNN: public InferenceHelper {
class InferenceWrapperRKNN : public InferenceWrapper {
public:
InferenceHelperRKNN();
~InferenceHelperRKNN() override;
InferenceWrapperRKNN();
~InferenceWrapperRKNN() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t SetCustomOps(const std::vector<std::pair<const char*, const void*>>& custom_ops) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list, std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
private:
rknn_context net_;
int32_t num_threads_;
@@ -42,9 +41,7 @@ private:
std::vector<std::string> input_names_;
private:
};
#endif // INFERENCE_HELPER_ENABLE_RKNN
#endif //RKNPU_PTOCR_INFERENCE_HELPER_RKNN_H
#endif // INFERENCE_WRAPPER_ENABLE_RKNN
#endif // RKNPU_PTOCR_INFERENCE_WRAPPER_RKNN_H

138
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn_adapter.cpp Normal file
View File

@@ -0,0 +1,138 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
#include <iostream>
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include <cassert>
#include "inference_wrapper_rknn_adapter.h"
#include "inference_wrapper_log.h"
#include "log.h"
#include <cassert>
#define TAG "InferenceWrapperRKNNAdapter"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceWrapperRKNNAdapter::InferenceWrapperRKNNAdapter() {
num_threads_ = 1;
}
InferenceWrapperRKNNAdapter::~InferenceWrapperRKNNAdapter() {}
int32_t InferenceWrapperRKNNAdapter::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::ParameterInitialization(std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::Process(std::vector<OutputTensorInfo> &output_tensor_info_list) {
if (output_tensor_info_list[0].tensor_type == TensorInfo::TensorTypeFp32) {
net_->setOutputsWantFloat(1);
// INSPIRE_LOGD("WANT FLOAT!");
}
auto ret = net_->RunModel();
if (ret != 0) {
INSPIRE_LOGE("Run model error.");
return WrapperError;
}
auto outputs_size = net_->GetOutputsNum();
assert(outputs_size == output_tensor_info_list.size());
for (int index = 0; index < outputs_size; ++index) {
auto &output_tensor = output_tensor_info_list[index];
output_tensor.data = net_->GetOutputFlow(index);
auto dim = net_->GetOutputTensorSize(index);
output_tensor.tensor_dims.clear();
for (int i = 0; i < dim.size(); ++i) {
output_tensor.tensor_dims.push_back((int)dim[i]);
// INSPIRE_LOGE("dim: %d", dim[i]);
}
}
net_->ReleaseOutputs();
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::PreProcess(const std::vector<InputTensorInfo> &input_tensor_info_list) {
for (int i = 0; i < input_tensor_info_list.size(); ++i) {
auto &input_tensor_info = input_tensor_info_list[i];
rknn_tensor_format fmt = RKNN_TENSOR_NHWC;
if (input_tensor_info.is_nchw) {
fmt = RKNN_TENSOR_NCHW;
} else {
fmt = RKNN_TENSOR_NHWC;
// INSPIRE_LOGD("NHWC!");
}
rknn_tensor_type type = RKNN_TENSOR_UINT8;
if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::TensorTypeFp32) {
type = RKNN_TENSOR_FLOAT32;
} else if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::TensorTypeUint8) {
type = RKNN_TENSOR_UINT8;
// INSPIRE_LOGD("UINT8!");
}
auto ret = net_->SetInputData(i, input_tensor_info.data, input_tensor_info.GetWidth(), input_tensor_info.GetHeight(),
input_tensor_info.GetChannel(), type, fmt);
if (ret != 0) {
INSPIRE_LOGE("Set data error.");
return ret;
}
}
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::Initialize(const std::string &model_filename, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
INSPIRE_LOGE("NOT IMPL");
return 0;
}
int32_t InferenceWrapperRKNNAdapter::Initialize(char *model_buffer, int model_size, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
net_ = std::make_shared<RKNNAdapter>();
auto ret = net_->Initialize((unsigned char *)model_buffer, model_size);
if (ret != 0) {
INSPIRE_LOGE("Rknn init error.");
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceWrapperRKNNAdapter::Finalize(void) {
if (net_ != nullptr) {
net_->Release();
}
return WrapperOk;
}
std::vector<std::string> InferenceWrapperRKNNAdapter::GetInputNames() {
return std::vector<std::string>();
}
int32_t InferenceWrapperRKNNAdapter::ResizeInput(const std::vector<InputTensorInfo> &input_tensor_info_list) {
// The function is not supported
return 0;
}
#endif // INFERENCE_WRAPPER_ENABLE_RKNN

45
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn_adapter.h Normal file
View File

@@ -0,0 +1,45 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_H
#define INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_H
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
#include "inference_wrapper.h"
#include "customized/rknn_adapter.h"
class InferenceWrapperRKNNAdapter : public InferenceWrapper {
public:
InferenceWrapperRKNNAdapter();
~InferenceWrapperRKNNAdapter() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
private:
std::shared_ptr<RKNNAdapter> net_;
int32_t num_threads_;
};
#endif // INFERENCE_WRAPPER_ENABLE_RKNN
#endif // INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_H

134
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn_adapter_nano.cpp Normal file
View File

@@ -0,0 +1,134 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN2
#include <iostream>
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include <cassert>
#include "inference_wrapper_rknn_adapter_nano.h"
#include "inference_wrapper_log.h"
#include "log.h"
#include <cassert>
#define TAG "InferenceWrapperRKNNAdapter"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
InferenceWrapperRKNNAdapter::InferenceWrapperRKNNAdapter() {
num_threads_ = 1;
}
InferenceWrapperRKNNAdapter::~InferenceWrapperRKNNAdapter() {}
int32_t InferenceWrapperRKNNAdapter::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::ParameterInitialization(std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::Process(std::vector<OutputTensorInfo> &output_tensor_info_list) {
if (output_tensor_info_list[0].tensor_type == TensorInfo::TensorTypeFp32) {
// net_->setOutputsWantFloat(1);
// INSPIRE_LOGD("WANT FLOAT!");
}
auto ret = net_->RunSession(true);
if (ret != 0) {
INSPIRE_LOGE("Run model error.");
return WrapperError;
}
auto outputs_size = net_->GetOutputAttrs().size();
assert(outputs_size == output_tensor_info_list.size());
for (int index = 0; index < outputs_size; ++index) {
auto &output_tensor = output_tensor_info_list[index];
output_tensor.data = (void *)net_->GetOutputDataPtr(index);
auto dim = net_->GetOutputTensorSize(index);
output_tensor.tensor_dims.clear();
for (int i = 0; i < dim.size(); ++i) {
output_tensor.tensor_dims.push_back((int)dim[i]);
// INSPIRE_LOGE("dim: %d", dim[i]);
}
}
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::PreProcess(const std::vector<InputTensorInfo> &input_tensor_info_list) {
for (int i = 0; i < input_tensor_info_list.size(); ++i) {
auto &input_tensor_info = input_tensor_info_list[i];
rknn_tensor_format fmt = RKNN_TENSOR_NHWC;
if (input_tensor_info.is_nchw) {
fmt = RKNN_TENSOR_NCHW;
} else {
fmt = RKNN_TENSOR_NHWC;
// INSPIRE_LOGD("NHWC!");
}
rknn_tensor_type type = RKNN_TENSOR_UINT8;
if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::TensorTypeFp32) {
type = RKNN_TENSOR_FLOAT32;
} else if (input_tensor_info.tensor_type == InputTensorInfo::TensorInfo::TensorTypeUint8) {
type = RKNN_TENSOR_UINT8;
// INSPIRE_LOGD("UINT8!");
}
auto ret = net_->SetInputData(i, (uint8_t *)input_tensor_info.data, type, fmt);
if (ret != 0) {
INSPIRE_LOGE("Set data error.");
return ret;
}
}
return WrapperOk;
}
int32_t InferenceWrapperRKNNAdapter::Initialize(const std::string &model_filename, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
INSPIRE_LOGE("NOT IMPL");
return 0;
}
int32_t InferenceWrapperRKNNAdapter::Initialize(char *model_buffer, int model_size, std::vector<InputTensorInfo> &input_tensor_info_list,
std::vector<OutputTensorInfo> &output_tensor_info_list) {
net_ = std::make_shared<RKNNAdapterNano>();
auto ret = net_->Initialize((unsigned char *)model_buffer, model_size);
if (ret != 0) {
INSPIRE_LOGE("Rknn init error.");
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
}
int32_t InferenceWrapperRKNNAdapter::Finalize(void) {
if (net_ != nullptr) {
net_->Release();
}
return WrapperOk;
}
std::vector<std::string> InferenceWrapperRKNNAdapter::GetInputNames() {
return std::vector<std::string>();
}
int32_t InferenceWrapperRKNNAdapter::ResizeInput(const std::vector<InputTensorInfo> &input_tensor_info_list) {
// The function is not supported
return 0;
}
#endif // INFERENCE_WRAPPER_ENABLE_RKNN2

45
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_rknn_adapter_nano.h Normal file
View File

@@ -0,0 +1,45 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_NANO_H
#define INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_NANO_H
#ifdef INFERENCE_WRAPPER_ENABLE_RKNN2
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
#include "inference_wrapper.h"
#include "customized/rknn_adapter_nano.h"
class InferenceWrapperRKNNAdapter : public InferenceWrapper {
public:
InferenceWrapperRKNNAdapter();
~InferenceWrapperRKNNAdapter() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
private:
std::shared_ptr<RKNNAdapterNano> net_;
int32_t num_threads_;
};
#endif // INFERENCE_WRAPPER_ENABLE_RKNN2
#endif // INSPIREFACE_INFERENCE_WRAPPER_RKNN_ADAPTER_NANO_H

208
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_tensorrt.cpp Normal file
View File

@@ -0,0 +1,208 @@
#if INFERENCE_WRAPPER_ENABLE_TENSORRT
#include <cstdint>
#include <cstdlib>
#include <cmath>
#include <cstring>
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include <MNN/ImageProcess.hpp>
#include "inference_wrapper_log.h"
#include "inference_wrapper_tensorrt.h"
#include "log.h"
#define TAG "InferenceWrapperTensorRT"
#define PRINT(...) INFERENCE_WRAPPER_LOG_PRINT(TAG, __VA_ARGS__)
#define PRINT_E(...) INFERENCE_WRAPPER_LOG_PRINT_E(TAG, __VA_ARGS__)
using namespace inspire;
InferenceWrapperTensorRT::InferenceWrapperTensorRT() {
num_threads_ = 1;
}
InferenceWrapperTensorRT::~InferenceWrapperTensorRT() {}
int32_t InferenceWrapperTensorRT::SetNumThreads(const int32_t num_threads) {
num_threads_ = num_threads;
return WrapperOk;
}
int32_t InferenceWrapperTensorRT::ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
return WrapperOk;
}
int32_t InferenceWrapperTensorRT::Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
net_.reset(new TensorRTAdapter());
net_->setDevice(device_id_);
auto ret = net_->readFromBin(model_buffer, model_size);
if (ret != WrapperOk) {
std::cout << "model_size: " << model_size << std::endl;
PRINT_E("Failed to load TensorRT model\n");
return WrapperError;
}
return WrapperOk;
}
int32_t InferenceWrapperTensorRT::Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) {
// LOG_INFO("init MNN");
/*** Create network ***/
net_.reset(new TensorRTAdapter());
net_->setDevice(device_id_);
auto ret = net_->readFromFile(model_filename);
if (ret != WrapperOk) {
PRINT_E("Failed to load model file (%s)\n", model_filename.c_str());
return WrapperError;
}
return ParameterInitialization(input_tensor_info_list, output_tensor_info_list);
};
int32_t InferenceWrapperTensorRT::Finalize(void) {
net_.reset();
return WrapperOk;
}
int32_t InferenceWrapperTensorRT::PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) {
// Currently only single-input models are supported
for (const auto& input_tensor_info : input_tensor_info_list) {
input_tensor_.reset(MNN::Tensor::create<float>(
std::vector<int>{1, 3, input_tensor_info.image_info.height, input_tensor_info.image_info.width}, nullptr, MNN::Tensor::CAFFE));
if (input_tensor_ == nullptr) {
PRINT_E("Invalid input name (%s)\n", input_tensor_info.name.c_str());
INSPIRE_LOGE("Invalid input name (%s)\n", input_tensor_info.name.c_str());
return WrapperError;
}
if (input_tensor_info.data_type == InputTensorInfo::DataTypeImage) {
/* Crop */
if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) ||
(input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) {
PRINT_E("Crop is not supported\n");
return WrapperError;
}
MNN::CV::ImageProcess::Config image_processconfig;
/* Convert color type */
// LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
// LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
// !!!!!! BUG !!!!!!!!!
// When initializing, setting the image channel to 3 and the tensor channel to 1,
// and configuring the processing to convert the color image to grayscale may cause some bugs.
// For example, the image channel might automatically change to 1.
// This issue has not been fully investigated,
// so it's necessary to manually convert the image to grayscale before input.
// !!!!!! BUG !!!!!!!!!
if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
if (input_tensor_info.image_info.swap_color) {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::RGB : MNN::CV::BGR;
} else {
image_processconfig.destFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
}
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::GRAY;
} else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) {
image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
image_processconfig.destFormat = MNN::CV::GRAY;
// LOGD("2gray");
} else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) {
image_processconfig.sourceFormat = MNN::CV::GRAY;
image_processconfig.destFormat = MNN::CV::BGR;
} else {
PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel());
return WrapperError;
}
/* Normalize image */
std::memcpy(image_processconfig.mean, input_tensor_info.normalize.mean, sizeof(image_processconfig.mean));
std::memcpy(image_processconfig.normal, input_tensor_info.normalize.norm, sizeof(image_processconfig.normal));
/* Resize image */
image_processconfig.filterType = MNN::CV::BILINEAR;
MNN::CV::Matrix trans;
trans.setScale(static_cast<float>(input_tensor_info.image_info.crop_width) / input_tensor_info.GetWidth(),
static_cast<float>(input_tensor_info.image_info.crop_height) / input_tensor_info.GetHeight());
/* Do pre-process */
std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(image_processconfig));
pretreat->setMatrix(trans);
// LOGD("k1");
pretreat->convert(static_cast<uint8_t*>(input_tensor_info.data), input_tensor_info.image_info.crop_width,
input_tensor_info.image_info.crop_height, 0, input_tensor_.get());
} else if ((input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) ||
(input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNchw)) {
std::unique_ptr<MNN::Tensor> tensor;
if (input_tensor_info.data_type == InputTensorInfo::DataTypeBlobNhwc) {
tensor.reset(new MNN::Tensor(input_tensor_.get(), MNN::Tensor::TENSORFLOW));
} else {
tensor.reset(new MNN::Tensor(input_tensor_.get(), MNN::Tensor::CAFFE));
}
if (tensor->getType().code == halide_type_float) {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<float>()[i] = static_cast<float*>(input_tensor_info.data)[i];
}
} else {
for (int32_t i = 0; i < input_tensor_info.GetWidth() * input_tensor_info.GetHeight() * input_tensor_info.GetChannel(); i++) {
tensor->host<uint8_t>()[i] = static_cast<uint8_t*>(input_tensor_info.data)[i];
}
}
input_tensor_->copyFromHostTensor(tensor.get());
} else {
PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type);
return WrapperError;
}
auto p = input_tensor_->host<float>();
net_->setInput(input_tensor_info.name.c_str(), reinterpret_cast<const char*>(p));
}
return WrapperOk;
}
int32_t InferenceWrapperTensorRT::Process(std::vector<OutputTensorInfo>& output_tensor_info_list) {
auto ret = net_->forward();
if (ret != TENSORRT_HSUCCEED) {
PRINT_E("Failed to forward\n");
return WrapperError;
}
// out_mat_list_.clear();
for (auto& output_tensor_info : output_tensor_info_list) {
// auto output_tensor = net_->getSessionOutput(session_, output_tensor_info.name.c_str());
const void* output_tensor = net_->getOutput(output_tensor_info.name.c_str());
if (output_tensor == nullptr) {
PRINT_E("Invalid output name (%s)\n", output_tensor_info.name.c_str());
return WrapperError;
}
output_tensor_info.data = (void*)output_tensor;
output_tensor_info.tensor_dims.clear();
const std::vector<int>& output_shape = net_->getOutputShapeByName(output_tensor_info.name);
int size = 1;
for (int32_t dim = 0; dim < output_shape.size(); dim++) {
output_tensor_info.tensor_dims.push_back(output_shape[dim]);
size *= output_shape[dim];
}
}
return WrapperOk;
}
std::vector<std::string> InferenceWrapperTensorRT::GetInputNames() {
return input_names_;
}
int32_t InferenceWrapperTensorRT::ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) {
PRINT_E("Currently, TensorRT does not support input resizing\n");
return 0;
}
#endif // INFERENCE_WRAPPER_ENABLE_TENSORRT

44
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/inference_wrapper_tensorrt.h Normal file
View File

@@ -0,0 +1,44 @@
#ifndef INFERENCE_WRAPPER_COREML_
#define INFERENCE_WRAPPER_COREML_
#if INFERENCE_WRAPPER_ENABLE_TENSORRT
#include <cstdint>
#include <cmath>
#include <string>
#include <vector>
#include <array>
#include <memory>
#include "tensorrt/tensorrt_adapter.h"
#include <MNN/ImageProcess.hpp>
#include "inference_wrapper.h"
class InferenceWrapperTensorRT : public InferenceWrapper {
public:
InferenceWrapperTensorRT();
~InferenceWrapperTensorRT() override;
int32_t SetNumThreads(const int32_t num_threads) override;
int32_t Initialize(const std::string& model_filename, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Initialize(char* model_buffer, int model_size, std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t Finalize(void) override;
int32_t PreProcess(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
int32_t Process(std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ParameterInitialization(std::vector<InputTensorInfo>& input_tensor_info_list,
std::vector<OutputTensorInfo>& output_tensor_info_list) override;
int32_t ResizeInput(const std::vector<InputTensorInfo>& input_tensor_info_list) override;
std::vector<std::string> GetInputNames() override;
private:
std::unique_ptr<TensorRTAdapter> net_;
int32_t num_threads_;
std::vector<std::string> input_names_;
/** Using MNN imageprocess to do Image Preprocessing */
std::unique_ptr<MNN::Tensor> input_tensor_;
};
#endif // INFERENCE_WRAPPER_COREML_
#endif // INFERENCE_WRAPPER_ENABLE_TENSORRT

657
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/tensorrt/tensorrt_adapter.cpp Normal file
View File

@@ -0,0 +1,657 @@
/**
* Created by Jingyu Yan
* @date 2025-03-16
*/
#if ISF_ENABLE_TENSORRT
#include "tensorrt_adapter.h"
#include <fstream>
#include <iostream>
#include <chrono>
#include <memory>
#include <cstring>
#include <cuda_fp16.h>
#include <NvInfer.h>
#include <cuda_runtime_api.h>
#include <log.h>
#include <isf_check.h>
// define specific deleters for TensorRT objects
struct TRTRuntimeDeleter {
void operator()(nvinfer1::IRuntime *runtime) const {
if (runtime)
delete runtime;
}
};
struct TRTEngineDeleter {
void operator()(nvinfer1::ICudaEngine *engine) const {
if (engine)
delete engine;
}
};
struct TRTContextDeleter {
void operator()(nvinfer1::IExecutionContext *context) const {
if (context)
delete context;
}
};
// custom deleter for CUDA stream
struct CUDAStreamDeleter {
void operator()(cudaStream_t *stream) const {
if (stream) {
cudaStreamDestroy(*stream);
delete stream;
}
}
};
// custom Logger class, inherit from TensorRT's ILogger
class TRTLogger : public nvinfer1::ILogger {
public:
void log(Severity severity, const char *msg) noexcept override {
if (severity <= Severity::kWARNING) {
INSPIRE_LOGI("[TensorRT] %s", msg);
}
}
};
// read model file to memory
static std::vector<char> readModelFile(const std::string &filename) {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
if (!file) {
INSPIRE_LOGE("failed to open model file: %s", filename.c_str());
return {};
}
size_t size = file.tellg();
file.seekg(0, std::ios::beg);
std::vector<char> buffer(size);
if (!file.read(buffer.data(), size)) {
INSPIRE_LOGE("failed to read model file: %s", filename.c_str());
return {};
}
return buffer;
}
// CUDA error check macro
#define CHECK_CUDA(call) \
do { \
cudaError_t error = call; \
if (error != cudaSuccess) { \
INSPIRE_LOGE("[CUDA error] %s", cudaGetErrorString(error)); \
return TENSORRT_HFAIL; \
} \
} while (0)
// TensorRT adapter implementation class
class TensorRTAdapter::Impl {
public:
Impl() : m_ownStream(false), m_inferenceMode(TensorRTAdapter::InferenceMode::FP32), m_deviceId(0) {
// create Logger with smart pointer
m_logger = std::make_unique<TRTLogger>();
}
int32_t initDevice() {
cudaError_t error = cudaSetDevice(m_deviceId);
if (error != cudaSuccess) {
INSPIRE_LOGE("[CUDA error] The device fails to use cuda:%d, %s", m_deviceId, cudaGetErrorString(error));
return TENSORRT_HFAIL;
}
return TENSORRT_HSUCCEED;
}
void setDevice(int32_t deviceId) {
m_deviceId = deviceId;
}
~Impl() {
// release resources - device memory needs to be released manually
for (auto &pair : m_deviceBuffers) {
if (pair.second) {
cudaFree(pair.second);
}
}
m_deviceBuffers.clear();
// smart pointers will handle the release of other resources
}
int32_t readFromFile(const std::string &enginePath) {
// read serialized engine file
std::vector<char> modelData = readModelFile(enginePath);
if (modelData.empty()) {
return TENSORRT_HFAIL;
}
return deserializeEngine(modelData);
}
int32_t readFromBin(const std::vector<char> &model_data) {
if (model_data.empty()) {
return TENSORRT_HFAIL;
}
return deserializeEngine(model_data);
}
int32_t readFromBin(void *model_data, unsigned int model_size) {
if (!model_data || model_size == 0) {
INSPIRE_LOGE("[TensorRT error] invalid model data or size");
return TENSORRT_HFAIL;
}
// convert memory data to vector to reuse the existing deserializeEngine method
std::vector<char> modelBuffer(static_cast<char *>(model_data), static_cast<char *>(model_data) + model_size);
return deserializeEngine(modelBuffer);
}
// create and deserialize engine
int32_t deserializeEngine(const std::vector<char> &modelData) {
// init device
initDevice();
// create runtime
m_runtime.reset(nvinfer1::createInferRuntime(*m_logger));
if (!m_runtime) {
INSPIRE_LOGE("[TensorRT error] failed to create TensorRT runtime");
return TENSORRT_HFAIL;
}
// deserialize engine
m_engine.reset(m_runtime->deserializeCudaEngine(modelData.data(), modelData.size()));
if (!m_engine) {
INSPIRE_LOGE("[TensorRT error] failed to deserialize engine");
return TENSORRT_HFAIL;
}
// create execution context
m_context.reset(m_engine->createExecutionContext());
if (!m_context) {
INSPIRE_LOGE("[TensorRT error] failed to create execution context");
return TENSORRT_HFAIL;
}
// get all input and output tensor names
int numIoTensors = m_engine->getNbIOTensors();
for (int i = 0; i < numIoTensors; ++i) {
const char *name = m_engine->getIOTensorName(i);
nvinfer1::TensorIOMode mode = m_engine->getTensorIOMode(name);
if (mode == nvinfer1::TensorIOMode::kINPUT) {
m_inputNames.push_back(name);
} else {
m_outputNames.push_back(name);
}
}
// initialize CUDA stream
if (!m_stream) {
cudaStream_t *stream = new cudaStream_t;
CHECK_CUDA(cudaStreamCreate(stream));
m_stream.reset(stream);
m_ownStream = true;
}
// pre-allocate device memory
return allocateDeviceMemory();
}
// allocate device memory
int32_t allocateDeviceMemory() {
// allocate device memory for each input and output tensor
for (const auto &name : m_inputNames) {
nvinfer1::Dims dims = m_engine->getTensorShape(name.c_str());
nvinfer1::DataType dtype = m_engine->getTensorDataType(name.c_str());
size_t size = getMemorySize(dims, dtype);
void *buffer = nullptr;
CHECK_CUDA(cudaMalloc(&buffer, size));
m_deviceBuffers[name] = buffer;
// store shape information
m_inputShapes[name] = dimsToVector(dims);
}
for (const auto &name : m_outputNames) {
nvinfer1::Dims dims = m_engine->getTensorShape(name.c_str());
nvinfer1::DataType dtype = m_engine->getTensorDataType(name.c_str());
size_t size = getMemorySize(dims, dtype);
void *buffer = nullptr;
CHECK_CUDA(cudaMalloc(&buffer, size));
m_deviceBuffers[name] = buffer;
// Save shape information
m_outputShapes[name] = dimsToVector(dims);
}
return TENSORRT_HSUCCEED;
}
// set input data
void setInput(const char *inputName, const void *data) {
auto it = m_deviceBuffers.find(inputName);
if (it != m_deviceBuffers.end()) {
nvinfer1::Dims dims = m_engine->getTensorShape(inputName);
nvinfer1::DataType dtype = m_engine->getTensorDataType(inputName);
size_t size = getMemorySize(dims, dtype);
// copy data from host to device
cudaMemcpyAsync(it->second, data, size, cudaMemcpyHostToDevice, *m_stream.get());
cudaStreamSynchronize(*m_stream.get()); // add synchronization to ensure data is fully copied
}
}
// set batch size (only for models with dynamic shapes)
int32_t setBatchSize(int batchSize) {
if (m_inputNames.empty())
return TENSORRT_HFAIL;
for (const auto &name : m_inputNames) {
nvinfer1::Dims dims = m_engine->getTensorShape(name.c_str());
if (dims.nbDims > 0) {
nvinfer1::Dims newDims = dims;
newDims.d[0] = batchSize;
if (!m_context->setInputShape(name.c_str(), newDims)) {
INSPIRE_LOGE("[TensorRT error] failed to set input shape for %s", name.c_str());
return TENSORRT_HFAIL;
}
// update shape information
m_inputShapes[name] = dimsToVector(newDims);
}
}
return TENSORRT_HSUCCEED;
}
// forward inference
int32_t forward() {
if (!m_context || !m_engine) {
return TENSORRT_HFAIL;
}
// check if all tensors are bound to addresses
for (const auto &name : m_inputNames) {
if (!m_context->setTensorAddress(name.c_str(), m_deviceBuffers[name])) {
INSPIRE_LOGE("[TensorRT error] failed to set input tensor %s address", name.c_str());
return TENSORRT_FORWARD_FAILED;
}
}
for (const auto &name : m_outputNames) {
if (!m_context->setTensorAddress(name.c_str(), m_deviceBuffers[name])) {
INSPIRE_LOGE("[TensorRT error] failed to set output tensor %s address", name.c_str());
return TENSORRT_FORWARD_FAILED;
}
}
// record start time - use high precision timing
auto start = std::chrono::high_resolution_clock::now();
// forward inference
bool status = m_context->enqueueV3(*m_stream.get());
// synchronize CUDA stream
cudaStreamSynchronize(*m_stream.get());
// record end time
auto end = std::chrono::high_resolution_clock::now();
// calculate duration (microseconds) then convert to milliseconds, keep high precision
auto duration_us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
m_inferenceTime = duration_us.count() / 1000.0;
return status ? TENSORRT_HSUCCEED : TENSORRT_FORWARD_FAILED;
}
// get output data
const void *getOutput(const char *nodeName) {
auto it = m_deviceBuffers.find(nodeName);
if (it != m_deviceBuffers.end()) {
nvinfer1::Dims dims = m_context->getTensorShape(nodeName);
nvinfer1::DataType dtype = m_engine->getTensorDataType(nodeName);
size_t size = getMemorySize(dims, dtype);
// copy output data from device to host
if (m_hostOutputBuffers.find(nodeName) == m_hostOutputBuffers.end()) {
m_hostOutputBuffers[nodeName].resize(size);
}
cudaMemcpyAsync(m_hostOutputBuffers[nodeName].data(), it->second, size, cudaMemcpyDeviceToHost, *m_stream.get());
cudaStreamSynchronize(*m_stream.get());
return m_hostOutputBuffers[nodeName].data();
}
return nullptr;
}
// get output data and convert to float type vector
std::vector<float> getOutputAsFloat(const char *nodeName) {
std::vector<float> result;
auto it = m_deviceBuffers.find(nodeName);
if (it != m_deviceBuffers.end()) {
nvinfer1::Dims dims = m_context->getTensorShape(nodeName);
nvinfer1::DataType dtype = m_engine->getTensorDataType(nodeName);
// calculate total number of elements
size_t numElements = 1;
for (int i = 0; i < dims.nbDims; ++i) {
numElements *= dims.d[i];
}
// allocate buffer of appropriate size based on data type
size_t elementSize = 0;
switch (dtype) {
case nvinfer1::DataType::kFLOAT:
elementSize = sizeof(float);
break;
case nvinfer1::DataType::kHALF:
elementSize = sizeof(half);
break;
case nvinfer1::DataType::kINT8:
elementSize = sizeof(int8_t);
break;
case nvinfer1::DataType::kINT32:
elementSize = sizeof(int32_t);
break;
default:
return result;
}
// allocate temporary buffer
std::vector<unsigned char> buffer(numElements * elementSize);
// copy data from device memory to host memory
cudaMemcpyAsync(buffer.data(), it->second, buffer.size(), cudaMemcpyDeviceToHost, *m_stream.get());
cudaStreamSynchronize(*m_stream.get());
// convert to float based on data type
result.resize(numElements);
switch (dtype) {
case nvinfer1::DataType::kFLOAT:
std::memcpy(result.data(), buffer.data(), buffer.size());
break;
case nvinfer1::DataType::kHALF: {
const half *halfData = reinterpret_cast<const half *>(buffer.data());
for (size_t i = 0; i < numElements; ++i) {
result[i] = __half2float(halfData[i]);
}
break;
}
case nvinfer1::DataType::kINT8: {
const int8_t *int8Data = reinterpret_cast<const int8_t *>(buffer.data());
for (size_t i = 0; i < numElements; ++i) {
result[i] = static_cast<float>(int8Data[i]);
}
break;
}
case nvinfer1::DataType::kINT32: {
const int32_t *int32Data = reinterpret_cast<const int32_t *>(buffer.data());
for (size_t i = 0; i < numElements; ++i) {
result[i] = static_cast<float>(int32Data[i]);
}
break;
}
}
}
return result;
}
// set inference mode
void setInferenceMode(TensorRTAdapter::InferenceMode mode) {
m_inferenceMode = mode;
// apply this setting during actual inference
}
// set CUDA stream
void setCudaStream(void *streamPtr) {
if (m_ownStream) {
m_stream.reset();
m_ownStream = false;
}
// create a new smart pointer instead of using reset + lambda
cudaStream_t *streamPointer = static_cast<cudaStream_t *>(streamPtr);
// use empty deleter, because this stream is managed by external code
m_stream =
std::unique_ptr<cudaStream_t, CUDAStreamDeleter>(streamPointer,
CUDAStreamDeleter() // use default deleter, but not actually delete the external stream
);
}
// print model info
void printModelInfo() const {
INSPIRE_LOGI("================================================");
if (!m_engine) {
INSPIRE_LOGE("[TensorRT error] engine not initialized");
return;
}
INSPIRE_LOGI("\nengine info:");
INSPIRE_LOGI("engine layers: %d", m_engine->getNbLayers());
INSPIRE_LOGI("input/output tensors: %d", m_engine->getNbIOTensors());
INSPIRE_LOGI("\ninput tensors:");
for (const auto &name : m_inputNames) {
nvinfer1::Dims dims = m_engine->getTensorShape(name.c_str());
nvinfer1::DataType dtype = m_engine->getTensorDataType(name.c_str());
INSPIRE_LOGI("name: %s, shape: (", name.c_str());
for (int d = 0; d < dims.nbDims; ++d) {
INSPIRE_LOGI("%d", dims.d[d]);
if (d < dims.nbDims - 1)
INSPIRE_LOGI(", ");
}
INSPIRE_LOGI("), type: %s", getDataTypeString(dtype).c_str());
}
INSPIRE_LOGI("\noutput tensors:");
for (const auto &name : m_outputNames) {
nvinfer1::Dims dims = m_engine->getTensorShape(name.c_str());
nvinfer1::DataType dtype = m_engine->getTensorDataType(name.c_str());
INSPIRE_LOGI("name: %s, shape: (", name.c_str());
for (int d = 0; d < dims.nbDims; ++d) {
INSPIRE_LOGI("%d", dims.d[d]);
if (d < dims.nbDims - 1)
INSPIRE_LOGI(", ");
}
INSPIRE_LOGI("), type: %s", getDataTypeString(dtype).c_str());
}
INSPIRE_LOGI("================================================");
}
// get input tensor names list
const std::vector<std::string> &getInputNames() const {
return m_inputNames;
}
// get output tensor names list
const std::vector<std::string> &getOutputNames() const {
return m_outputNames;
}
// get input tensor shape by name
const std::vector<int> &getInputShapeByName(const std::string &name) const {
static std::vector<int> emptyShape;
auto it = m_inputShapes.find(name);
return (it != m_inputShapes.end()) ? it->second : emptyShape;
}
// get output tensor shape by name
const std::vector<int> &getOutputShapeByName(const std::string &name) const {
static std::vector<int> emptyShape;
auto it = m_outputShapes.find(name);
return (it != m_outputShapes.end()) ? it->second : emptyShape;
}
// get inference time
double getInferenceTime() const {
return m_inferenceTime;
}
private:
// helper function: convert TensorRT's Dims to standard vector
std::vector<int> dimsToVector(const nvinfer1::Dims &dims) const {
std::vector<int> shape;
for (int i = 0; i < dims.nbDims; ++i) {
shape.push_back(dims.d[i]);
}
return shape;
}
// helper function: calculate memory size
size_t getMemorySize(const nvinfer1::Dims &dims, nvinfer1::DataType dtype) const {
size_t size = 1;
for (int i = 0; i < dims.nbDims; ++i) {
size *= dims.d[i];
}
switch (dtype) {
case nvinfer1::DataType::kFLOAT:
return size * 4;
case nvinfer1::DataType::kHALF:
return size * 2;
case nvinfer1::DataType::kINT8:
return size;
case nvinfer1::DataType::kINT32:
return size * 4;
case nvinfer1::DataType::kBOOL:
return size;
default:
return size;
}
}
// helper function: get data type string representation
std::string getDataTypeString(nvinfer1::DataType dtype) const {
switch (dtype) {
case nvinfer1::DataType::kFLOAT:
return "FLOAT";
case nvinfer1::DataType::kHALF:
return "HALF";
case nvinfer1::DataType::kINT8:
return "INT8";
case nvinfer1::DataType::kINT32:
return "INT32";
case nvinfer1::DataType::kBOOL:
return "BOOL";
default:
return "UNKNOWN";
}
}
// member variables - using smart pointers
std::unique_ptr<TRTLogger> m_logger;
std::unique_ptr<nvinfer1::IRuntime, TRTRuntimeDeleter> m_runtime;
std::unique_ptr<nvinfer1::ICudaEngine, TRTEngineDeleter> m_engine;
std::unique_ptr<nvinfer1::IExecutionContext, TRTContextDeleter> m_context;
bool m_ownStream;
std::unique_ptr<cudaStream_t, CUDAStreamDeleter> m_stream;
int32_t m_deviceId{0};
std::vector<std::string> m_inputNames;
std::vector<std::string> m_outputNames;
std::map<std::string, void *> m_deviceBuffers;
std::map<std::string, std::vector<unsigned char>> m_hostOutputBuffers;
std::map<std::string, std::vector<int>> m_inputShapes;
std::map<std::string, std::vector<int>> m_outputShapes;
TensorRTAdapter::InferenceMode m_inferenceMode;
double m_inferenceTime;
};
// implement TensorRTAdapter methods
TensorRTAdapter::TensorRTAdapter() : pImpl(new Impl()) {}
TensorRTAdapter::~TensorRTAdapter() {
if (pImpl) {
delete pImpl;
pImpl = nullptr;
}
}
int32_t TensorRTAdapter::readFromFile(const std::string &enginePath) {
return pImpl->readFromFile(enginePath);
}
int32_t TensorRTAdapter::readFromBin(void *model_data, unsigned int model_size) {
return pImpl->readFromBin(model_data, model_size);
}
TensorRTAdapter TensorRTAdapter::readNetFrom(const std::string &enginePath) {
TensorRTAdapter adapter;
adapter.readFromFile(enginePath);
return adapter;
}
TensorRTAdapter TensorRTAdapter::readNetFromBin(const std::vector<char> &model_data) {
TensorRTAdapter adapter;
adapter.pImpl->readFromBin(model_data);
return adapter;
}
std::vector<std::string> TensorRTAdapter::getInputNames() const {
return pImpl->getInputNames();
}
std::vector<std::string> TensorRTAdapter::getOutputNames() const {
return pImpl->getOutputNames();
}
std::vector<int> TensorRTAdapter::getInputShapeByName(const std::string &name) {
return pImpl->getInputShapeByName(name);
}
std::vector<int> TensorRTAdapter::getOutputShapeByName(const std::string &name) {
return pImpl->getOutputShapeByName(name);
}
void TensorRTAdapter::setInput(const char *inputName, const void *data) {
pImpl->setInput(inputName, data);
}
int32_t TensorRTAdapter::setBatchSize(int batchSize) {
return pImpl->setBatchSize(batchSize);
}
int32_t TensorRTAdapter::forward() {
return pImpl->forward();
}
const void *TensorRTAdapter::getOutput(const char *nodeName) {
return pImpl->getOutput(nodeName);
}
std::vector<float> TensorRTAdapter::getOutputAsFloat(const char *nodeName) {
return pImpl->getOutputAsFloat(nodeName);
}
double TensorRTAdapter::getInferenceTime() const {
return pImpl->getInferenceTime();
}
void TensorRTAdapter::setInferenceMode(InferenceMode mode) {
pImpl->setInferenceMode(mode);
}
void TensorRTAdapter::setCudaStream(void *streamPtr) {
pImpl->setCudaStream(streamPtr);
}
void TensorRTAdapter::printModelInfo() const {
pImpl->printModelInfo();
}
void TensorRTAdapter::setDevice(int32_t deviceId) {
pImpl->setDevice(deviceId);
}
#endif // ISF_ENABLE_TENSORRT

175
cpp-package/inspireface/cpp/inspireface/middleware/inference_wrapper/tensorrt/tensorrt_adapter.h Normal file
View File

@@ -0,0 +1,175 @@
/**
* Created by Jingyu Yan
* @date 2025-03-16
*/
#if ISF_ENABLE_TENSORRT
#ifndef INSPIRE_TENSORRT_ADAPTER_H
#define INSPIRE_TENSORRT_ADAPTER_H
#include <string>
#include <vector>
#include <map>
#define TENSORRT_HSUCCEED 0
#define TENSORRT_HFAIL -1
#define TENSORRT_FORWARD_FAILED -2
/**
* @brief The TensorRT adapter is used for inference
*/
class TensorRTAdapter {
public:
/**
* @brief inference mode (abeyance)
*/
enum class InferenceMode {
FP32, ///< FP32 precision inference
FP16, ///< FP16 precision inference
INT8 ///< INT8 precision inference (requires calibration)
};
/**
* @brief output shape mapping
*/
typedef std::map<std::string, std::vector<int>> TensorShapesMap;
/**
* @brief default constructor
*/
TensorRTAdapter();
/**
* @brief destructor
*/
~TensorRTAdapter();
/**
* @brief read model from file
* @param enginePath engine file path
* @return 0 means success, -1 means failure
*/
int32_t readFromFile(const std::string &enginePath);
/**
* @brief read model from binary data
* @param model_data binary model data
* @param model_size model data size
* @return 0 means success, -1 means failure
*/
int32_t readFromBin(void *model_data, unsigned int model_size);
/**
* @brief read model from file static method
* @param enginePath engine file path
* @return TensorRTAdapter instance
*/
static TensorRTAdapter readNetFrom(const std::string &enginePath);
/**
* @brief read model from binary data static method
* @param model_data binary model data
* @return TensorRTAdapter instance
*/
static TensorRTAdapter readNetFromBin(const std::vector<char> &model_data);
/**
* @brief get all input tensor names
* @return input tensor names list
*/
std::vector<std::string> getInputNames() const;
/**
* @brief get all output tensor names
* @return output tensor names list
*/
std::vector<std::string> getOutputNames() const;
/**
* @brief get input tensor shape by name
* @param name tensor name
* @return shape vector
*/
std::vector<int> getInputShapeByName(const std::string &name);
/**
* @brief get output tensor shape by name
* @param name tensor name
* @return shape vector
*/
std::vector<int> getOutputShapeByName(const std::string &name);
/**
* @brief set input data
* @param inputName input tensor name
* @param data input data
*/
void setInput(const char *inputName, const void *data);
/**
* @brief set dynamic batch size
* @param batchSize batch size
* @return 0 means success, -1 means failure
*/
int32_t setBatchSize(int batchSize);
/**
* @brief forward inference
* @return 0 means success, -1 means failure
*/
int32_t forward();
/**
* @brief get output data
* @param nodeName output tensor name
* @return output data pointer
*/
const void *getOutput(const char *nodeName);
/**
* @brief get output data and convert to float type
* @param nodeName output tensor name
* @return output data pointer
*/
std::vector<float> getOutputAsFloat(const char *nodeName);
/**
* @brief get inference time (ms)
* @return inference time
*/
double getInferenceTime() const;
/**
* @brief set inference mode
* @param mode inference mode
*/
void setInferenceMode(InferenceMode mode);
/**
* @brief set CUDA stream
* @param stream CUDA stream
*/
void setCudaStream(void *streamPtr);
/**
* @brief print model info
*/
void printModelInfo() const;
/**
* @brief set CUDA device
* @param deviceId CUDA device id
*/
void setDevice(int32_t deviceId);
private:
// use PIMPL pattern to hide implementation details
class Impl;
Impl *pImpl;
// output shape cache
TensorShapesMap m_inputShapes;
TensorShapesMap m_outputShapes;
};
#endif // INSPIRE_TENSORRT_ADAPTER_H
#endif // ISF_ENABLE_TENSORRT

392
cpp-package/inspireface/cpp/inspireface/middleware/inspirecv_image_process.h Normal file
View File

@@ -0,0 +1,392 @@
#ifndef INSPIRECV_IMAGE_PROCESS_H
#define INSPIRECV_IMAGE_PROCESS_H
#include <memory>
#include <inspirecv/inspirecv.h>
#include <MNN/ImageProcess.hpp>
#include "isf_check.h"
// using namespace inspire;
namespace inspirecv {
/**
* @brief Enum to represent rotation modes.
*/
enum ROTATION_MODE { ROTATION_0 = 0, ROTATION_90 = 1, ROTATION_180 = 2, ROTATION_270 = 3 };
/**
* @brief Enum to represent data formats.
*/
enum DATA_FORMAT { NV21 = 0, NV12 = 1, RGBA = 2, RGB = 3, BGR = 4, BGRA = 5 };
/**
* @brief A class to handle camera stream and image processing.
*/
class InspireImageProcess {
public:
static InspireImageProcess Create(const uint8_t *data_buffer, int height, int width, DATA_FORMAT data_format = BGR,
ROTATION_MODE rotation_mode = ROTATION_0) {
InspireImageProcess process;
process.SetDataBuffer(data_buffer, height, width);
process.SetDataFormat(data_format);
process.SetRotationMode(rotation_mode);
return process;
}
InspireImageProcess() {
SetDataFormat(NV21);
SetDestFormat(BGR);
config_.filterType = MNN::CV::BILINEAR;
config_.wrap = MNN::CV::ZERO;
rotation_mode_ = ROTATION_0;
preview_size_ = 192;
UpdateTransformMatrix();
}
/**
* @brief Set the data buffer, height, and width of the camera stream.
*
* @param data_buffer Pointer to the data buffer.
* @param height Height of the image.
* @param width Width of the image.
*/
void SetDataBuffer(const uint8_t *data_buffer, int height, int width) {
this->buffer_ = data_buffer;
this->height_ = height;
this->width_ = width;
preview_scale_ = preview_size_ / static_cast<float>(std::max(height, width));
UpdateTransformMatrix();
}
/**
* @brief Set the preview size.
*
* @param size Preview size.
*/
void SetPreviewSize(const int size) {
preview_size_ = size;
preview_scale_ = preview_size_ / static_cast<float>(std::max(this->height_, this->width_));
UpdateTransformMatrix();
}
void SetPreviewScale(const float scale) {
preview_scale_ = scale;
preview_size_ = static_cast<int>(preview_scale_ * std::max(this->height_, this->width_));
UpdateTransformMatrix();
}
/**
* @brief Set the rotation mode.
*
* @param mode Rotation mode (e.g., ROTATION_0, ROTATION_90).
*/
void SetRotationMode(ROTATION_MODE mode) {
rotation_mode_ = mode;
UpdateTransformMatrix();
}
/**
* @brief Set the data format.
*
* @param data_format Data format (e.g., NV21, RGBA).
*/
void SetDataFormat(DATA_FORMAT data_format) {
if (data_format == NV21) {
config_.sourceFormat = MNN::CV::YUV_NV21;
}
if (data_format == NV12) {
config_.sourceFormat = MNN::CV::YUV_NV12;
}
if (data_format == RGBA) {
config_.sourceFormat = MNN::CV::RGBA;
}
if (data_format == RGB) {
config_.sourceFormat = MNN::CV::RGB;
}
if (data_format == BGR) {
config_.sourceFormat = MNN::CV::BGR;
}
if (data_format == BGRA) {
config_.sourceFormat = MNN::CV::BGRA;
}
}
/**
* @brief Set the destination format.
*
* @param data_format Data format (e.g., NV21, RGBA).
*/
void SetDestFormat(DATA_FORMAT data_format) {
if (data_format == NV21) {
config_.destFormat = MNN::CV::YUV_NV21;
}
if (data_format == NV12) {
config_.destFormat = MNN::CV::YUV_NV12;
}
if (data_format == RGBA) {
config_.destFormat = MNN::CV::RGBA;
}
if (data_format == RGB) {
config_.destFormat = MNN::CV::RGB;
}
if (data_format == BGR) {
config_.destFormat = MNN::CV::BGR;
}
if (data_format == BGRA) {
config_.destFormat = MNN::CV::BGRA;
}
}
/**
* @brief Get an affine-transformed image.
*
* @param affine_matrix Affine transformation matrix.
* @param width_out Width of the output image.
* @param height_out Height of the output image.
* @return cv::Mat Affine-transformed image.
*/
inspirecv::Image ExecuteImageAffineProcessing(inspirecv::TransformMatrix &affine_matrix, const int width_out, const int height_out) const {
int sw = width_;
int sh = height_;
int rot_sw = sw;
int rot_sh = sh;
MNN::CV::Matrix tr;
std::vector<float> tr_cv({1, 0, 0, 0, 1, 0, 0, 0, 1});
memcpy(tr_cv.data(), affine_matrix.Squeeze().data(), sizeof(float) * 6);
tr.set9(tr_cv.data());
MNN::CV::Matrix tr_inv;
tr.invert(&tr_inv);
std::shared_ptr<MNN::CV::ImageProcess> process(MNN::CV::ImageProcess::create(config_));
process->setMatrix(tr_inv);
auto img_out = inspirecv::Image::Create(width_out, height_out, 3);
std::shared_ptr<MNN::Tensor> tensor(MNN::Tensor::create<uint8_t>(std::vector<int>{1, height_out, width_out, 3}, (uint8_t *)img_out.Data()));
auto ret = process->convert(buffer_, sw, sh, 0, tensor.get());
INSPIREFACE_CHECK_MSG(ret == MNN::ErrorCode::NO_ERROR, "ImageProcess::convert failed");
return img_out;
}
/**
* @brief Get a preview image with optional rotation.
*
* @param with_rotation True if rotation is applied, false otherwise.
* @return cv::Mat Preview image.
*/
inspirecv::Image ExecutePreviewImageProcessing(bool with_rotation) {
return ExecuteImageScaleProcessing(preview_scale_, with_rotation);
}
/**
* @brief Get the preview scale.
*
* @return float Preview scale.
*/
float GetPreviewScale() {
return preview_scale_;
}
/**
* @brief Execute image scale processing.
*
* @param scale Scale factor.
* @param with_rotation True if rotation is applied, false otherwise.
* @return inspirecv::Image Scaled image.
*/
inspirecv::Image ExecuteImageScaleProcessing(const float scale, bool with_rotation) {
int sw = width_;
int sh = height_;
int rot_sw = sw;
int rot_sh = sh;
// MNN::CV::Matrix tr;
std::shared_ptr<MNN::CV::ImageProcess> process(MNN::CV::ImageProcess::create(config_));
if (rotation_mode_ == ROTATION_270 && with_rotation) {
float srcPoints[] = {
0.0f, 0.0f, 0.0f, (float)(height_ - 1), (float)(width_ - 1), 0.0f, (float)(width_ - 1), (float)(height_ - 1),
};
float dstPoints[] = {(float)(height_ * scale - 1), 0.0f, 0.0f, 0.0f, (float)(height_ * scale - 1), (float)(width_ * scale - 1), 0.0f,
(float)(width_ * scale - 1)};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints, (MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(width_ * scale);
int scaled_width = static_cast<int>(height_ * scale);
inspirecv::Image img_out(scaled_width, scaled_height, 3);
std::shared_ptr<MNN::Tensor> tensor(
MNN::Tensor::create<uint8_t>(std::vector<int>{1, scaled_height, scaled_width, 3}, (uint8_t *)img_out.Data()));
auto ret = process->convert(buffer_, sw, sh, 0, tensor.get());
INSPIREFACE_CHECK_MSG(ret == MNN::ErrorCode::NO_ERROR, "ImageProcess::convert failed");
return img_out;
} else if (rotation_mode_ == ROTATION_90 && with_rotation) {
float srcPoints[] = {
0.0f, 0.0f, 0.0f, (float)(height_ - 1), (float)(width_ - 1), 0.0f, (float)(width_ - 1), (float)(height_ - 1),
};
float dstPoints[] = {
0.0f, (float)(width_ * scale - 1), (float)(height_ * scale - 1), (float)(width_ * scale - 1), 0.0f, 0.0f, (float)(height_ * scale - 1),
0.0f,
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints, (MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(width_ * scale);
int scaled_width = static_cast<int>(height_ * scale);
inspirecv::Image img_out(scaled_width, scaled_height, 3);
std::shared_ptr<MNN::Tensor> tensor(
MNN::Tensor::create<uint8_t>(std::vector<int>{1, scaled_height, scaled_width, 3}, (uint8_t *)img_out.Data()));
auto ret = process->convert(buffer_, sw, sh, 0, tensor.get());
INSPIREFACE_CHECK_MSG(ret == MNN::ErrorCode::NO_ERROR, "ImageProcess::convert failed");
return img_out;
} else if (rotation_mode_ == ROTATION_180 && with_rotation) {
float srcPoints[] = {
0.0f, 0.0f, 0.0f, (float)(height_ - 1), (float)(width_ - 1), 0.0f, (float)(width_ - 1), (float)(height_ - 1),
};
float dstPoints[] = {
(float)(width_ * scale - 1),
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
0.0f,
(float)(height_ * scale - 1),
0.0f,
0.0f,
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints, (MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(height_ * scale);
int scaled_width = static_cast<int>(width_ * scale);
inspirecv::Image img_out(scaled_width, scaled_height, 3);
std::shared_ptr<MNN::Tensor> tensor(
MNN::Tensor::create<uint8_t>(std::vector<int>{1, scaled_height, scaled_width, 3}, (uint8_t *)img_out.Data()));
auto ret = process->convert(buffer_, sw, sh, 0, tensor.get());
INSPIREFACE_CHECK_MSG(ret == MNN::ErrorCode::NO_ERROR, "ImageProcess::convert failed");
return img_out;
} else {
float srcPoints[] = {
0.0f, 0.0f, 0.0f, (float)(height_ - 1), (float)(width_ - 1), 0.0f, (float)(width_ - 1), (float)(height_ - 1),
};
float dstPoints[] = {
0.0f,
0.0f,
0.0f,
(float)(height_ * scale - 1),
(float)(width_ * scale - 1),
0.0f,
(float)(width_ * scale - 1),
(float)(height_ * scale - 1),
};
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints, (MNN::CV::Point *)srcPoints, 4);
process->setMatrix(tr_);
int scaled_height = static_cast<int>(height_ * scale);
int scaled_width = static_cast<int>(width_ * scale);
inspirecv::Image img_out(scaled_width, scaled_height, 3);
std::shared_ptr<MNN::Tensor> tensor(
MNN::Tensor::create<uint8_t>(std::vector<int>{1, scaled_height, scaled_width, 3}, (uint8_t *)img_out.Data()));
auto ret = process->convert(buffer_, sw, sh, 0, tensor.get());
INSPIREFACE_CHECK_MSG(ret == MNN::ErrorCode::NO_ERROR, "ImageProcess::convert failed");
return img_out;
}
}
inspirecv::TransformMatrix GetAffineMatrix() const {
auto affine_matrix = inspirecv::TransformMatrix::Create();
affine_matrix[0] = tr_[0];
affine_matrix[1] = tr_[1];
affine_matrix[2] = tr_[2];
affine_matrix[3] = tr_[3];
affine_matrix[4] = tr_[4];
affine_matrix[5] = tr_[5];
return affine_matrix;
}
/**
* @brief Get the height of the camera stream image.
*
* @return int Height.
*/
int GetHeight() const {
return height_;
}
/**
* @brief Get the width of the camera stream image.
*
* @return int Width.
*/
int GetWidth() const {
return width_;
}
/**
* @brief Get the current rotation mode.
*
* @return ROTATION_MODE Current rotation mode.
*/
ROTATION_MODE getRotationMode() const {
return rotation_mode_;
}
private:
void UpdateTransformMatrix() {
float srcPoints[] = {0.0f, 0.0f, 0.0f, (float)(height_ - 1), (float)(width_ - 1), 0.0f, (float)(width_ - 1), (float)(height_ - 1)};
float dstPoints[8];
if (rotation_mode_ == ROTATION_270) {
float points[] = {(float)(height_ * preview_scale_ - 1),
0.0f,
0.0f,
0.0f,
(float)(height_ * preview_scale_ - 1),
(float)(width_ * preview_scale_ - 1),
0.0f,
(float)(width_ * preview_scale_ - 1)};
memcpy(dstPoints, points, sizeof(points));
} else if (rotation_mode_ == ROTATION_90) {
float points[] = {0.0f,
(float)(width_ * preview_scale_ - 1),
(float)(height_ * preview_scale_ - 1),
(float)(width_ * preview_scale_ - 1),
0.0f,
0.0f,
(float)(height_ * preview_scale_ - 1),
0.0f};
memcpy(dstPoints, points, sizeof(points));
} else if (rotation_mode_ == ROTATION_180) {
float points[] = {(float)(width_ * preview_scale_ - 1),
(float)(height_ * preview_scale_ - 1),
(float)(width_ * preview_scale_ - 1),
0.0f,
0.0f,
(float)(height_ * preview_scale_ - 1),
0.0f,
0.0f};
memcpy(dstPoints, points, sizeof(points));
} else { // ROTATION_0
float points[] = {0.0f,
0.0f,
0.0f,
(float)(height_ * preview_scale_ - 1),
(float)(width_ * preview_scale_ - 1),
0.0f,
(float)(width_ * preview_scale_ - 1),
(float)(height_ * preview_scale_ - 1)};
memcpy(dstPoints, points, sizeof(points));
}
tr_.setPolyToPoly((MNN::CV::Point *)dstPoints, (MNN::CV::Point *)srcPoints, 4);
}
private:
const uint8_t *buffer_; ///< Pointer to the data buffer.
int buffer_size_; ///< Size of the data buffer.
std::vector<float> rotation_matrix; ///< Rotation matrix.
int height_; ///< Height of the camera stream image.
int width_; ///< Width of the camera stream image.
float preview_scale_; ///< Scaling factor for the preview image.
int preview_size_; ///< Size of the preview image.
MNN::CV::Matrix tr_; ///< Affine transformation matrix.
ROTATION_MODE rotation_mode_; ///< Current rotation mode.
MNN::CV::ImageProcess::Config config_; ///< Configuration for image processing.
std::shared_ptr<MNN::CV::ImageProcess> process_; ///< Image processing instance.
};
} // namespace inspirecv
#endif // INSPIRECV_IMAGE_PROCESS_H

171
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/core_archive/core_archive.cc Normal file
View File

@@ -0,0 +1,171 @@
/**
* Created by Jingyu Yan
* @date 2025-03-23
*/
#include "core_archive.h"
#include "microtar/microtar.h"
#include "log.h"
#include <unordered_map>
#include <iostream>
namespace inspire {
class CoreArchive::Impl {
public:
Impl() : m_load_file_status_(SARC_NOT_LOAD), m_tar_(nullptr) {}
explicit Impl(const std::string& archiveFile) : m_load_file_status_(SARC_NOT_LOAD), m_tar_(nullptr) {
Reset(archiveFile);
}
~Impl() {
Close();
}
int32_t Reset(const std::string& archiveFile) {
Close();
std::vector<char>().swap(m_empty_);
m_tar_ = std::make_shared<mtar_t>();
m_load_file_status_ = mtar_open(m_tar_.get(), archiveFile.c_str(), "r");
if (m_load_file_status_ != MTAR_ESUCCESS) {
INSPIRE_LOGE("Invalid archive file: %d", m_load_file_status_);
Close();
m_tar_.reset();
return m_load_file_status_;
}
mtar_header_t h;
m_load_file_status_ = mtar_read_header(m_tar_.get(), &h);
if (m_load_file_status_ != MTAR_ESUCCESS) {
INSPIRE_LOGE("Error reading root from archive.");
return m_load_file_status_;
}
// m_file_archive_root_ = std::string(h.name);
size_t index = 0;
while ((mtar_read_header(m_tar_.get(), &h)) != MTAR_ENULLRECORD) {
m_load_file_status_ = mtar_next(m_tar_.get());
if (m_load_file_status_ != MTAR_ESUCCESS) {
INSPIRE_LOGE("Failed to scan the file: %d", m_load_file_status_);
Close();
return m_load_file_status_;
}
m_subfiles_names_.emplace_back(h.name);
index++;
}
return m_load_file_status_;
}
std::vector<char>& GetFileContent(const std::string& filename) {
auto index = filenameFuzzyMatching(filename);
if (index != std::string::npos) {
auto fullFilename = m_subfiles_names_[index];
auto ret = lazyReadFile(fullFilename);
if (ret != MTAR_ESUCCESS) {
INSPIRE_LOGE("Failed to load file");
}
return m_file_content_cache_map_[fullFilename];
}
return m_empty_;
}
int32_t QueryLoadStatus() const {
return m_load_file_status_;
}
const std::vector<std::string>& GetSubfilesNames() const {
return m_subfiles_names_;
}
void Close() {
if (m_tar_ && m_tar_.get() != nullptr) {
mtar_close(m_tar_.get());
}
m_tar_.reset();
m_load_file_status_ = SARC_NOT_LOAD;
m_subfiles_names_.clear();
m_file_content_cache_map_.clear();
}
void PrintSubFiles() {
std::cout << "Subfiles: " << m_subfiles_names_.size() << std::endl;
for (int i = 0; i < m_subfiles_names_.size(); ++i) {
std::cout << m_subfiles_names_[i] << std::endl;
}
}
private:
size_t filenameFuzzyMatching(const std::string& filename) {
for (size_t i = 0; i < m_subfiles_names_.size(); ++i) {
if (m_subfiles_names_[i].find(filename) != std::string::npos) {
return i;
}
}
return std::string::npos;
}
int32_t lazyReadFile(const std::string& filename) {
if (m_file_content_cache_map_.find(filename) != m_file_content_cache_map_.end()) {
return MTAR_ESUCCESS;
}
mtar_header_t h;
auto ret = mtar_find(m_tar_.get(), filename.c_str(), &h);
if (ret == MTAR_ESUCCESS) {
std::vector<char> content(h.size);
ret = mtar_read_data(m_tar_.get(), content.data(), h.size);
if (ret == MTAR_ESUCCESS) {
m_file_content_cache_map_[filename] = std::move(content); // Load and store the file contents
return MTAR_ESUCCESS;
} else {
INSPIRE_LOGE("Failed to load file: %d", ret);
}
} else {
INSPIRE_LOGE("Failed to find file: %d", ret);
}
return SARC_LOAD_FILE_FAIL;
}
std::string m_file_archive_root_; ///< Archive file path
std::vector<std::string> m_subfiles_names_; ///< Name list of subfiles
std::shared_ptr<mtar_t> m_tar_; ///< mtar context
int32_t m_load_file_status_; ///< Initiation status code
std::vector<char> m_empty_; ///< Const empty
std::unordered_map<std::string, std::vector<char>> m_file_content_cache_map_; ///< File buffer cache
};
CoreArchive::CoreArchive() : m_pImpl(std::make_unique<Impl>()) {}
CoreArchive::CoreArchive(const std::string& archiveFile) : m_pImpl(std::make_unique<Impl>(archiveFile)) {}
CoreArchive::~CoreArchive() = default;
CoreArchive::CoreArchive(CoreArchive&& other) noexcept = default;
CoreArchive& CoreArchive::operator=(CoreArchive&& other) noexcept = default;
int32_t CoreArchive::Reset(const std::string& archiveFile) {
return m_pImpl->Reset(archiveFile);
}
std::vector<char>& CoreArchive::GetFileContent(const std::string& filename) {
return m_pImpl->GetFileContent(filename);
}
int32_t CoreArchive::QueryLoadStatus() const {
return m_pImpl->QueryLoadStatus();
}
const std::vector<std::string>& CoreArchive::GetSubfilesNames() const {
return m_pImpl->GetSubfilesNames();
}
void CoreArchive::Close() {
m_pImpl->Close();
}
void CoreArchive::PrintSubFiles() {
m_pImpl->PrintSubFiles();
}
} // namespace inspire

61
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/core_archive/core_archive.h Normal file
View File

@@ -0,0 +1,61 @@
/**
* Created by Jingyu Yan
* @date 2025-03-23
*/
#pragma once
#ifndef MODELLOADERTAR_CPP_MICROTAR_H
#define MODELLOADERTAR_CPP_MICROTAR_H
#ifndef INSPIRE_API
#define INSPIRE_API
#endif
#include <string>
#include <vector>
#include <memory>
namespace inspire {
enum {
SARC_SUCCESS = 0,
SARC_FAILURE = -1,
SARC_OPEN_FAIL = -2,
SARC_READ_FAIL = -3,
SARC_WRITE_FAIL = -4,
SARC_SEEK_FAIL = -5,
SARC_BAD_CHKSUM = -6,
SARC_NULL_RECORD = -7,
SARC_NOTFOUND = -8,
SARC_LOAD_FILE_FAIL = -9,
SARC_NOT_LOAD = -10,
};
class INSPIRE_API CoreArchive {
public:
explicit CoreArchive(const std::string& archiveFile);
explicit CoreArchive();
~CoreArchive();
// Copy construction and assignment operations are disabled
CoreArchive(const CoreArchive&) = delete;
CoreArchive& operator=(const CoreArchive&) = delete;
// Enable mobile construction and assignment operations
CoreArchive(CoreArchive&& other) noexcept;
CoreArchive& operator=(CoreArchive&& other) noexcept;
int32_t Reset(const std::string& archiveFile);
std::vector<char>& GetFileContent(const std::string& filename);
int32_t QueryLoadStatus() const;
const std::vector<std::string>& GetSubfilesNames() const;
void Close();
void PrintSubFiles();
private:
class Impl;
std::unique_ptr<Impl> m_pImpl;
}; // class CoreArchive
} // namespace inspire
#endif // MODELLOADERTAR_CPP_MICROTAR_H

42
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/microtar/microtar.c → cpp-package/inspireface/cpp/inspireface/middleware/model_archive/core_archive/microtar/microtar.c
View File

@@ -169,7 +169,9 @@ static int file_seek(mtar_t *tar, unsigned offset) {
}
static int file_close(mtar_t *tar) {
fclose(tar->stream);
if (tar->source_type == FROM_FILE) {
fclose(tar->stream);
}
return MTAR_ESUCCESS;
}
@@ -184,7 +186,7 @@ int mtar_open(mtar_t *tar, const char *filename, const char *mode) {
tar->read = file_read;
tar->seek = file_seek;
tar->close = file_close;
tar->source_type = FROM_FILE;
/* Assure mode is always binary */
if ( strchr(mode, 'r') ) mode = "rb";
if ( strchr(mode, 'w') ) mode = "wb";
@@ -374,3 +376,39 @@ int mtar_finalize(mtar_t *tar) {
/* Write two NULL records */
return write_null_bytes(tar, sizeof(mtar_raw_header_t) * 2);
}
static int memory_read(mtar_t *tar, void *data, unsigned size) {
if (tar->pos + size > tar->stream_size) {
return MTAR_EREADFAIL;
}
memcpy(data, (char *)tar->stream + tar->pos, size);
tar->pos += size;
return MTAR_ESUCCESS;
}
static int memory_seek(mtar_t *tar, unsigned pos) {
if (pos > tar->stream_size) {
return MTAR_ESEEKFAIL;
}
tar->pos = pos;
return MTAR_ESUCCESS;
}
int mtar_open_memory(mtar_t *tar, void *data, size_t size) {
memset(tar, 0, sizeof(*tar));
tar->read = memory_read;
tar->seek = memory_seek;
tar->stream = data;
tar->stream_size = size; // Add a field to store the data size
tar->source_type = FROM_MEMORY;
tar->close = file_close;
// Read the first header to verify the data
mtar_header_t h;
int err = mtar_read_header(tar, &h);
if (err != MTAR_ESUCCESS) {
return err;
}
return mtar_rewind(tar);
}

70
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/microtar/microtar.h → cpp-package/inspireface/cpp/inspireface/middleware/model_archive/core_archive/microtar/microtar.h
View File

@@ -9,66 +9,58 @@
#define MICROTAR_H
#ifdef __cplusplus
extern "C"
{
extern "C" {
#endif
#include <stdio.h>
#include <stdlib.h>
#define MTAR_VERSION "0.1.0"
#define MTAR_VERSION "0.2.0"
enum {
MTAR_ESUCCESS = 0,
MTAR_EFAILURE = -1,
MTAR_EOPENFAIL = -2,
MTAR_EREADFAIL = -3,
MTAR_EWRITEFAIL = -4,
MTAR_ESEEKFAIL = -5,
MTAR_EBADCHKSUM = -6,
MTAR_ENULLRECORD = -7,
MTAR_ENOTFOUND = -8
MTAR_ESUCCESS = 0,
MTAR_EFAILURE = -1,
MTAR_EOPENFAIL = -2,
MTAR_EREADFAIL = -3,
MTAR_EWRITEFAIL = -4,
MTAR_ESEEKFAIL = -5,
MTAR_EBADCHKSUM = -6,
MTAR_ENULLRECORD = -7,
MTAR_ENOTFOUND = -8
};
enum {
MTAR_TREG = '0',
MTAR_TLNK = '1',
MTAR_TSYM = '2',
MTAR_TCHR = '3',
MTAR_TBLK = '4',
MTAR_TDIR = '5',
MTAR_TFIFO = '6'
};
enum { MTAR_TREG = '0', MTAR_TLNK = '1', MTAR_TSYM = '2', MTAR_TCHR = '3', MTAR_TBLK = '4', MTAR_TDIR = '5', MTAR_TFIFO = '6' };
typedef struct {
unsigned mode;
unsigned owner;
unsigned size;
unsigned mtime;
unsigned type;
char name[100];
char linkname[100];
unsigned mode;
unsigned owner;
unsigned size;
unsigned mtime;
unsigned type;
char name[100];
char linkname[100];
} mtar_header_t;
typedef struct mtar_t mtar_t;
struct mtar_t {
int (*read)(mtar_t *tar, void *data, unsigned size);
int (*write)(mtar_t *tar, const void *data, unsigned size);
int (*seek)(mtar_t *tar, unsigned pos);
int (*close)(mtar_t *tar);
void *stream;
unsigned pos;
unsigned remaining_data;
unsigned last_header;
int (*read)(mtar_t *tar, void *data, unsigned size);
int (*write)(mtar_t *tar, const void *data, unsigned size);
int (*seek)(mtar_t *tar, unsigned pos);
int (*close)(mtar_t *tar);
void *stream;
unsigned pos;
unsigned stream_size; // Added a field to record the size of a memory stream
unsigned remaining_data;
unsigned last_header;
enum { FROM_FILE, FROM_MEMORY } source_type;
};
const char* mtar_strerror(int err);
const char *mtar_strerror(int err);
int mtar_open(mtar_t *tar, const char *filename, const char *mode);
int mtar_close(mtar_t *tar);
int mtar_open_memory(mtar_t *tar, void *data, size_t size);
int mtar_seek(mtar_t *tar, unsigned pos);
int mtar_rewind(mtar_t *tar);

62
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/simple_archive.h → cpp-package/inspireface/cpp/inspireface/middleware/model_archive/core_archive/simple_archive.h
View File

@@ -1,6 +1,7 @@
//
// Created by tunm on 2024/3/29.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef MODELLOADERTAR_CPP_MICROTAR_H
#define MODELLOADERTAR_CPP_MICROTAR_H
@@ -16,17 +17,17 @@
namespace inspire {
enum {
SARC_SUCCESS = 0,
SARC_FAILURE = -1,
SARC_OPEN_FAIL = -2,
SARC_READ_FAIL = -3,
SARC_WRITE_FAIL = -4,
SARC_SEEK_FAIL = -5,
SARC_BAD_CHKSUM = -6,
SARC_NULL_RECORD = -7,
SARC_NOTFOUND = -8,
SARC_SUCCESS = 0,
SARC_FAILURE = -1,
SARC_OPEN_FAIL = -2,
SARC_READ_FAIL = -3,
SARC_WRITE_FAIL = -4,
SARC_SEEK_FAIL = -5,
SARC_BAD_CHKSUM = -6,
SARC_NULL_RECORD = -7,
SARC_NOTFOUND = -8,
SARC_LOAD_FILE_FAIL = -9,
SARC_NOT_LOAD = -10,
SARC_NOT_LOAD = -10,
};
class INSPIRE_API SimpleArchive {
@@ -40,11 +41,13 @@ public:
}
int32_t Reset(const std::string& archiveFile) {
Close();
std::vector<char>().swap(m_empty_);
m_tar_ = std::make_shared<mtar_t>();
m_load_file_status_ = mtar_open(m_tar_.get(), archiveFile.c_str(), "r");
if (m_load_file_status_ != MTAR_ESUCCESS) {
INSPIRE_LOGE("Invalid archive file: %d", m_load_file_status_);
Close();
m_tar_.reset();
return m_load_file_status_;
}
@@ -54,12 +57,13 @@ public:
INSPIRE_LOGE("Error reading root from archive.");
return m_load_file_status_;
}
// m_file_archive_root_ = std::string(h.name);
// m_file_archive_root_ = std::string(h.name);
size_t index = 0;
while ( (mtar_read_header(m_tar_.get(), &h)) != MTAR_ENULLRECORD ) {
while ((mtar_read_header(m_tar_.get(), &h)) != MTAR_ENULLRECORD) {
m_load_file_status_ = mtar_next(m_tar_.get());
if (m_load_file_status_ != MTAR_ESUCCESS) {
INSPIRE_LOGE("Failed to scan the file");
INSPIRE_LOGE("Failed to scan the file: %d", m_load_file_status_);
Close();
return m_load_file_status_;
}
m_subfiles_names_.emplace_back(h.name);
@@ -85,14 +89,13 @@ public:
return m_load_file_status_;
}
const std::vector<std::string> &GetSubfilesNames() const {
const std::vector<std::string>& GetSubfilesNames() const {
return m_subfiles_names_;
}
~SimpleArchive() {
~SimpleArchive() {
Close();
}
void Close() {
if (m_tar_.get() != nullptr) {
@@ -102,7 +105,6 @@ public:
m_load_file_status_ = SARC_NOT_LOAD;
m_subfiles_names_.clear();
}
void PrintSubFiles() {
std::cout << "Subfiles: " << m_subfiles_names_.size() << std::endl;
@@ -112,7 +114,6 @@ public:
}
protected:
size_t filenameFuzzyMatching(const std::string& filename) {
for (size_t i = 0; i < m_subfiles_names_.size(); ++i) {
if (m_subfiles_names_[i].find(filename) != std::string::npos) {
@@ -132,7 +133,7 @@ protected:
std::vector<char> content(h.size);
ret = mtar_read_data(m_tar_.get(), content.data(), h.size);
if (ret == MTAR_ESUCCESS) {
m_file_content_cache_map_[filename] = std::move(content); // Load and store the file contents
m_file_content_cache_map_[filename] = std::move(content); // Load and store the file contents
return MTAR_ESUCCESS;
} else {
INSPIRE_LOGE("Failed to load file: %d", ret);
@@ -145,17 +146,16 @@ protected:
}
private:
std::string m_file_archive_root_; ///< Archive file path
std::vector<std::string> m_subfiles_names_; ///< Name list of subfiles
std::shared_ptr<mtar_t> m_tar_; ///< mtar context
int32_t m_load_file_status_; ///< Initiation status code
std::string m_file_archive_root_; ///< Archive file path
std::vector<std::string> m_subfiles_names_; ///< Name list of subfiles
std::shared_ptr<mtar_t> m_tar_; ///< mtar context
int32_t m_load_file_status_; ///< Initiation status code
std::vector<char> m_empty_; ///< Const empty
std::vector<char> m_empty_; ///< Const empty
std::unordered_map<std::string, std::vector<char>> m_file_content_cache_map_; ///< File buffer cache
std::unordered_map<std::string, std::vector<char>> m_file_content_cache_map_; ///< File buffer cache
};
} // namespace inspire
} // namespace inspire
#endif //MODELLOADERTAR_CPP_MICROTAR_H
#endif // MODELLOADERTAR_CPP_MICROTAR_H

113
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/inspire_archive.h
View File

@@ -1,13 +1,16 @@
//
// Created by tunm on 2024/3/30.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef MODELLOADERTAR_INSPIREARCHIVE_H
#define MODELLOADERTAR_INSPIREARCHIVE_H
#include "simple_archive.h"
#include "core_archive/core_archive.h"
#include "inspire_model/inspire_model.h"
#include "yaml-cpp/yaml.h"
#include "fstream"
#include "recognition_module/similarity_converter.h"
namespace inspire {
@@ -19,22 +22,46 @@ enum {
NOT_READ = -15,
};
class INSPIRE_API InspireArchive: SimpleArchive {
class INSPIRE_API InspireArchive {
public:
InspireArchive() : SimpleArchive() {
InspireArchive() : m_archive_(std::make_shared<CoreArchive>()) {
m_status_ = NOT_READ;
}
explicit InspireArchive(const std::string& archiveFile) : SimpleArchive(archiveFile) {
m_status_ = QueryStatus();
explicit InspireArchive(const std::string& archiveFile) : m_archive_(std::make_shared<CoreArchive>(archiveFile)) {
m_status_ = m_archive_->QueryLoadStatus();
if (m_status_ == SARC_SUCCESS) {
m_status_ = loadManifestFile();
}
}
InspireArchive(const InspireArchive& other)
: m_archive_(other.m_archive_),
m_config_(other.m_config_),
m_status_(other.m_status_),
m_tag_(other.m_tag_),
m_version_(other.m_version_),
m_major_(other.m_major_),
m_release_time_(other.m_release_time_) {}
InspireArchive& operator=(const InspireArchive& other) {
if (this != &other) {
m_archive_ = other.m_archive_;
m_config_ = other.m_config_;
m_status_ = other.m_status_;
m_tag_ = other.m_tag_;
m_version_ = other.m_version_;
m_major_ = other.m_major_;
m_release_time_ = other.m_release_time_;
}
return *this;
}
int32_t ReLoad(const std::string& archiveFile) {
auto ret = Reset(archiveFile);
auto ret = m_archive_->Reset(archiveFile);
if (ret != SARC_SUCCESS) {
m_archive_->Close();
m_status_ = ret;
return ret;
}
m_status_ = loadManifestFile();
@@ -45,13 +72,17 @@ public:
return m_status_;
}
int32_t LoadModel(const std::string &name, InspireModel &model) {
int32_t LoadModel(const std::string& name, InspireModel& model) {
if (m_config_[name]) {
auto ret = model.Reset(m_config_[name]);
if (ret != 0) {
return ret;
}
auto &buffer = GetFileContent(model.name);
if (model.loadFilePath) {
// No model files are loaded, only configuration files are loaded for extension modules such as CoreML.
return SARC_SUCCESS;
}
auto& buffer = m_archive_->GetFileContent(model.name);
if (buffer.empty()) {
return ERROR_MODEL_BUFFER;
}
@@ -62,20 +93,27 @@ public:
}
}
void PublicPrintSubFiles() {
PrintSubFiles();
void PrintSubFiles() {
m_archive_->PrintSubFiles();
}
const std::vector<std::string>& GetSubfilesNames() const {
return m_archive_->GetSubfilesNames();
}
void Release() {
m_status_ = NOT_READ;
Close();
m_archive_->Close();
}
std::vector<char>& GetFileContent(const std::string& filename) {
return m_archive_->GetFileContent(filename);
}
private:
int32_t loadManifestFile() {
if (QueryLoadStatus() == SARC_SUCCESS) {
auto configBuffer = GetFileContent(MANIFEST_FILE);
if (m_archive_->QueryLoadStatus() == SARC_SUCCESS) {
auto configBuffer = m_archive_->GetFileContent(MANIFEST_FILE);
configBuffer.push_back('\0');
if (configBuffer.empty()) {
return MISS_MANIFEST;
@@ -86,12 +124,44 @@ private:
}
m_tag_ = m_config_["tag"].as<std::string>();
m_version_ = m_config_["version"].as<std::string>();
INSPIRE_LOGI("== %s %s ==", m_tag_.c_str(), m_version_.c_str());
if (m_config_["major"]) {
m_major_ = m_config_["major"].as<std::string>();
} else {
m_major_ = "unknown";
}
if (m_config_["release"]) {
m_release_time_ = m_config_["release"].as<std::string>();
} else {
m_release_time_ = "unknown";
}
INSPIRE_LOGI("== Load %s-%s, Version: %s, Release: %s ==", m_tag_.c_str(), m_major_.c_str(), m_version_.c_str(), m_release_time_.c_str());
// Load similarity converter config
if (m_config_["similarity_converter"]) {
SimilarityConverterConfig config;
config.threshold = m_config_["similarity_converter"]["threshold"].as<double>();
config.middleScore = m_config_["similarity_converter"]["middle_score"].as<double>();
config.steepness = m_config_["similarity_converter"]["steepness"].as<double>();
config.outputMin = m_config_["similarity_converter"]["output_min"].as<double>();
config.outputMax = m_config_["similarity_converter"]["output_max"].as<double>();
SIMILARITY_CONVERTER_UPDATE_CONFIG(config);
INSPIRE_LOGI(
"Successfully loaded similarity converter config: \n \t threshold: %f \n \t middle_score: %f \n \t steepness: %f \n \t output_min: "
"%f \n \t output_max: %f",
config.threshold, config.middleScore, config.steepness, config.outputMin, config.outputMax);
SIMILARITY_CONVERTER_SET_RECOMMENDED_COSINE_THRESHOLD(config.threshold);
} else {
INSPIRE_LOGW("No similarity converter config found, use default config: ");
auto config = SIMILARITY_CONVERTER_GET_CONFIG();
INSPIRE_LOGI("threshold: %f \n \t middle_score: %f \n \t steepness: %f \n \t output_min: %f \n \t output_max: %f", config.threshold,
config.middleScore, config.steepness, config.outputMin, config.outputMax);
SIMILARITY_CONVERTER_SET_RECOMMENDED_COSINE_THRESHOLD(config.threshold);
}
}
return 0;
}
private:
std::shared_ptr<CoreArchive> m_archive_;
YAML::Node m_config_;
int32_t m_status_;
@@ -100,9 +170,10 @@ private:
std::string m_tag_;
std::string m_version_;
std::string m_major_;
std::string m_release_time_;
};
} // namespace inspire
} // namespace inspire
#endif //MODELLOADERTAR_INSPIREARCHIVE_H
#endif // MODELLOADERTAR_INSPIREARCHIVE_H

128
cpp-package/inspireface/cpp/inspireface/middleware/model_archive/inspire_model/inspire_model.h
View File

@@ -1,6 +1,7 @@
//
// Created by tunm on 2024/4/2.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#ifndef MODELLOADERTAR_INSPIRE_MODEL_H
#define MODELLOADERTAR_INSPIRE_MODEL_H
@@ -9,12 +10,26 @@
#include "yaml-cpp/yaml.h"
#include "middleware/configurable.h"
#include "log.h"
#include "middleware/inference_helper/inference_helper.h"
#include "middleware/inference_wrapper/inference_wrapper.h"
namespace inspire {
typedef enum {
InspireInferBackendAuto = 10,
InspireInferBackendCPU = 0,
InspireInferBackendRKNPU = 1,
InspireInferBackendCUDA = 2,
} InspireInferBackend;
typedef enum {
InspireInferEngineMNN = 0,
InspireInferEngineRKNN = 1,
InspireInferEngineCoreML = 2,
InspireInferEngineTensorRT = 3,
} InspireInferEngine;
class INSPIRE_API InspireModel {
CONFIGURABLE_SUPPORT
CONFIGURABLE_SUPPORT
public:
explicit InspireModel(const YAML::Node &node) {
Reset(node);
@@ -37,33 +52,51 @@ public:
if (node["model_type"]) {
auto type = node["model_type"].as<std::string>();
if (type == "MNN") {
modelType = InferenceHelper::kMnn;
modelType = InferenceWrapper::INFER_MNN;
} else if (type == "RKNN") {
modelType = InferenceHelper::kRknn;
modelType = InferenceWrapper::INFER_RKNN;
} else if (type == "COREML") {
modelType = InferenceWrapper::INFER_COREML;
// Special handling, the binary model is not loaded by default
loadFilePath = 1;
} else if (type == "TensorRT") {
modelType = InferenceWrapper::INFER_TENSORRT;
}
}
if (node["infer_engine"]) {
auto type = node["infer_engine"].as<std::string>();
if (type == "MNN") {
inferEngine = InferenceHelper::kMnn;
inferEngine = InferenceWrapper::INFER_MNN;
} else if (type == "RKNN") {
inferEngine = InferenceHelper::kRknn;
inferEngine = InferenceWrapper::INFER_RKNN;
} else if (type == "COREML") {
inferEngine = InferenceWrapper::INFER_COREML;
} else if (type == "TensorRT") {
inferEngine = InferenceWrapper::INFER_TENSORRT;
}
}
if (node["infer_device"]) {
auto type = node["infer_device"].as<std::string>();
if (type == "MNN") {
inferDevice = 0;
inferDevice = InspireInferEngineMNN;
} else if (type == "RKNPU") {
inferDevice = 1;
inferDevice = InspireInferEngineRKNN;
} else if (type == "COREML") {
inferDevice = InspireInferEngineCoreML;
} else if (type == "CUDA") {
inferDevice = InspireInferEngineTensorRT;
}
}
if (node["infer_backend"]) {
auto type = node["infer_backend"].as<std::string>();
if (type == "CPU") {
inferBackend = 0;
inferBackend = InspireInferBackendCPU;
} else if (type == "RKNPU") {
inferBackend = 1;
inferBackend = InspireInferBackendRKNPU;
} else if (type == "AUTO") {
inferBackend = InspireInferBackendAuto;
} else if (type == "CUDA") {
inferBackend = InspireInferBackendCUDA;
}
}
return decode(node);
@@ -73,17 +106,16 @@ public:
INSPIRE_LOGD("%s", m_configuration.toString().c_str());
}
void SetBuffer(std::vector<char>& modelBuffer, size_t size) {
void SetBuffer(std::vector<char> &modelBuffer, size_t size) {
buffer = modelBuffer.data();
bufferSize = size;
}
Configurable& Config() {
Configurable &Config() {
return m_configuration;
}
private:
int32_t decode(const YAML::Node &node) {
try {
if (node["input_channel"]) {
@@ -101,43 +133,43 @@ private:
if (node["data_type"]) {
auto type = node["data_type"].as<std::string>();
if (type == "image") {
setData<int>("data_type", InputTensorInfo::InputTensorInfo::kDataTypeImage);
} else if(type == "data_nhwc") {
setData<int>("data_type", InputTensorInfo::InputTensorInfo::kDataTypeBlobNhwc);
} else if(type == "data_nchw") {
setData<int>("data_type", InputTensorInfo::InputTensorInfo::kDataTypeBlobNchw);
setData<int>("data_type", InputTensorInfo::InputTensorInfo::DataTypeImage);
} else if (type == "data_nhwc") {
setData<int>("data_type", InputTensorInfo::InputTensorInfo::DataTypeBlobNhwc);
} else if (type == "data_nchw") {
setData<int>("data_type", InputTensorInfo::InputTensorInfo::DataTypeBlobNchw);
}
}
if (node["input_tensor_type"]) {
auto type = node["input_tensor_type"].as<std::string>();
if (type == "none") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeNone);
} else if(type == "uint8") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeUint8);
} else if(type == "int8") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt8);
} else if(type == "float32") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeFp32);
} else if(type == "int32") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt32);
} else if(type == "int64") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt64);
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeNone);
} else if (type == "uint8") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeUint8);
} else if (type == "int8") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt8);
} else if (type == "float32") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeFp32);
} else if (type == "int32") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt32);
} else if (type == "int64") {
setData<int>("input_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt64);
}
}
if (node["output_tensor_type"]) {
auto type = node["output_tensor_type"].as<std::string>();
if (type == "none") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeNone);
} else if(type == "uint8") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeUint8);
} else if(type == "int8") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt8);
} else if(type == "float32") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeFp32);
} else if(type == "int32") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt32);
} else if(type == "int64") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::kTensorTypeInt64);
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeNone);
} else if (type == "uint8") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeUint8);
} else if (type == "int8") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt8);
} else if (type == "float32") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeFp32);
} else if (type == "int32") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt32);
} else if (type == "int64") {
setData<int>("output_tensor_type", InputTensorInfo::TensorInfo::TensorTypeInt64);
}
}
if (node["threads"]) {
@@ -178,7 +210,7 @@ private:
}
setData<std::vector<float>>("norm", norm);
}
} catch (const YAML::Exception& e) {
} catch (const YAML::Exception &e) {
INSPIRE_LOGE("An error occurred parsing the interpretation file in archive: %s", e.what());
return -1;
}
@@ -189,16 +221,16 @@ public:
std::string name;
std::string fullname;
std::string version;
InferenceHelper::HelperType modelType;
InferenceWrapper::EngineType modelType;
int inferEngine;
int inferDevice;
int inferBackend;
int loadFilePath{0};
char *buffer;
size_t bufferSize;
};
}; // namespace inspire
}; // namespace inspire
#endif //MODELLOADERTAR_INSPIRE_MODEL_H
#endif // MODELLOADERTAR_INSPIRE_MODEL_H

25
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor.cpp Normal file
View File

@@ -0,0 +1,25 @@
#include "image_processor.h"
#if defined(ISF_ENABLE_RGA)
#include "image_processor_rga.h"
#else
#include "image_processor_general.h"
#endif
namespace inspire {
namespace nexus {
ImageProcessor::~ImageProcessor() = default;
std::unique_ptr<ImageProcessor> ImageProcessor::Create() {
#if defined(ISF_ENABLE_RGA)
return std::make_unique<RgaImageProcessor>();
#else
return std::make_unique<GeneralImageProcessor>();
#endif
}
} // namespace nexus
} // namespace inspire

60
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor.h Normal file
View File

@@ -0,0 +1,60 @@
#ifndef INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_H
#define INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_H
#include <iostream>
#include <vector>
#include <inspirecv/inspirecv.h>
#include <memory>
namespace inspire {
namespace nexus {
/**
* @brief Extensible image processing interface that supports hardware acceleration backends
*
* This interface provides common image processing operations like resize, color conversion,
* padding etc. It can be implemented by different backends based on compile options:
* - Default CPU-based implementation using InspireCV (always available)
* - Hardware accelerated implementation like Rockchip RGA (enabled with ISF_ENABLE_RGA)
* - Other potential hardware acceleration backends can be enabled via corresponding compile flags
*
* The backend implementation is selected at compile time based on which acceleration options
* are enabled. Only one backend will be active at runtime.
*/
class ImageProcessor {
public:
static std::unique_ptr<ImageProcessor> Create();
public:
// Virtual destructor
virtual ~ImageProcessor() = 0;
// Resize image to specified dimensions
virtual int32_t Resize(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data, int dst_width,
int dst_height) = 0;
// Swap color channels of the image
virtual int32_t SwapColor(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data) = 0;
// Add padding to image borders
virtual int32_t Padding(const uint8_t* src_data, int src_width, int src_height, int channels, int top, int bottom, int left, int right,
uint8_t** dst_data, int& dst_width, int& dst_height) = 0;
// Resize image and add padding as needed
virtual int32_t ResizeAndPadding(const uint8_t* src_data, int src_width, int src_height, int channels, int dst_width, int dst_height,
uint8_t** dst_data, float& scale) = 0;
// Mark processing as complete
virtual int32_t MarkDone() = 0;
// Display cache status information
virtual void DumpCacheStatus() const = 0;
}; // class ImageProcessor
} // namespace nexus
} // namespace inspire
#endif // INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_H

59
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor_general.cpp Normal file
View File

@@ -0,0 +1,59 @@
#include "image_processor_general.h"
namespace inspire {
namespace nexus {
int32_t GeneralImageProcessor::Resize(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data, int dst_width,
int dst_height) {
inspirecv::Image src_img(src_width, src_height, channels, src_data, false);
last_buffer_.image = src_img.Resize(dst_width, dst_height);
*dst_data = last_buffer_.GetData();
return 0;
}
int32_t GeneralImageProcessor::SwapColor(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data) {
inspirecv::Image src_img(src_width, src_height, channels, src_data, false);
last_buffer_.image = src_img.SwapRB();
*dst_data = last_buffer_.GetData();
return 0;
}
int32_t GeneralImageProcessor::Padding(const uint8_t* src_data, int src_width, int src_height, int channels, int top, int bottom, int left, int right,
uint8_t** dst_data, int& dst_width, int& dst_height) {
inspirecv::Image src_img(src_width, src_height, channels, src_data, false);
dst_width = src_width + left + right;
dst_height = src_height + top + bottom;
last_buffer_.image = src_img.Pad(top, bottom, left, right, inspirecv::Color::Black);
*dst_data = last_buffer_.GetData();
return 0;
}
int32_t GeneralImageProcessor::ResizeAndPadding(const uint8_t* src_data, int src_width, int src_height, int channels, int dst_width, int dst_height,
uint8_t** dst_data, float& scale) {
inspirecv::Image src_img(src_width, src_height, channels, src_data, false);
scale = std::min(static_cast<float>(dst_width) / src_width, static_cast<float>(dst_height) / src_height);
int resized_w = static_cast<int>(src_width * scale);
int resized_h = static_cast<int>(src_height * scale);
int wpad = dst_width - resized_w;
int hpad = dst_height - resized_h;
inspirecv::Image resized_img = src_img.Resize(resized_w, resized_h);
last_buffer_.image = resized_img.Pad(0, hpad, 0, wpad, inspirecv::Color::Black);
*dst_data = last_buffer_.GetData();
return 0;
}
int32_t GeneralImageProcessor::MarkDone() {
return 0;
}
void GeneralImageProcessor::DumpCacheStatus() const {
INSPIRECV_LOG(INFO) << "GeneralImageProcessor has no cache to dump";
}
} // namespace nexus
} // namespace inspire

45
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor_general.h Normal file
View File

@@ -0,0 +1,45 @@
#ifndef INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_GENERAL_H
#define INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_GENERAL_H
#include "image_processor.h"
#include <inspirecv/inspirecv.h>
namespace inspire {
namespace nexus {
class GeneralImageProcessor : public ImageProcessor {
public:
GeneralImageProcessor() = default;
~GeneralImageProcessor() override = default;
int32_t Resize(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data, int dst_width, int dst_height) override;
int32_t SwapColor(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data) override;
int32_t Padding(const uint8_t* src_data, int src_width, int src_height, int channels, int top, int bottom, int left, int right,
uint8_t** dst_data, int& dst_width, int& dst_height) override;
int32_t ResizeAndPadding(const uint8_t* src_data, int src_width, int src_height, int channels, int dst_width, int dst_height, uint8_t** dst_data,
float& scale) override;
int32_t MarkDone() override;
void DumpCacheStatus() const override;
private:
struct BufferWrapper {
inspirecv::Image image;
uint8_t* GetData() {
return const_cast<uint8_t*>(image.Data());
}
};
BufferWrapper last_buffer_;
}; // GeneralImageProcessor
} // namespace nexus
} // namespace inspire
#endif // INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_GENERAL_H

245
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor_rga.cpp Normal file
View File

@@ -0,0 +1,245 @@
#include "image_processor_rga.h"
#if defined(ISF_ENABLE_RGA)
namespace inspire {
namespace nexus {
RgaImageProcessor::RgaImageProcessor() {}
RgaImageProcessor::~RgaImageProcessor() {}
int32_t RgaImageProcessor::Resize(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data, int dst_width,
int dst_height) {
// Calculate width aligned to 4 bytes
int aligned_src_width = (src_width + 3) & ~3; // Round up to nearest multiple of 4
int aligned_dst_width = (dst_width + 3) & ~3;
// 1. Get or create source buffer with aligned width
BufferKey src_key{aligned_src_width, src_height, channels};
auto& src_buffer = GetOrCreateBuffer(src_key);
// 2. Get or create destination buffer with aligned width
BufferKey dst_key{aligned_dst_width, dst_height, channels};
auto& dst_buffer = GetOrCreateBuffer(dst_key, false);
// 3. Copy source data to RGA buffer, handle padding row by row
for (int y = 0; y < src_height; y++) {
memcpy(static_cast<uint8_t*>(src_buffer.virtual_addr) + y * aligned_src_width * channels, src_data + y * src_width * channels,
src_width * channels);
// Padding area remains zero
}
dma_sync_cpu_to_device(src_buffer.dma_fd);
dma_sync_cpu_to_device(dst_buffer.dma_fd);
// 4. Execute RGA resize
int ret = imcheck(src_buffer.buffer, dst_buffer.buffer, {}, {});
if (IM_STATUS_NOERROR != ret) {
INSPIRECV_LOG(ERROR) << "RGA parameter check failed: " << imStrError((IM_STATUS)ret);
return false;
}
ret = imresize(src_buffer.buffer, dst_buffer.buffer);
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA resize failed: " << imStrError((IM_STATUS)ret);
return false;
}
// 5. Return pointer to destination buffer
*dst_data = static_cast<uint8_t*>(dst_buffer.virtual_addr);
return 0;
}
int32_t RgaImageProcessor::MarkDone() {
// Sync all buffers
for (const auto& pair : buffer_cache_) {
dma_sync_device_to_cpu(pair.second.dma_fd);
}
// // Print current cache status for debugging
// INSPIRECV_LOG(INFO) << "MarkDone: Current cache status:";
// INSPIRECV_LOG(INFO) << "Cache size: " << buffer_cache_.size();
// for (const auto& pair : buffer_cache_) {
// INSPIRECV_LOG(INFO) << "Buffer: " << pair.second.width << "x" << pair.second.height << " dma_fd=" << pair.second.dma_fd;
// }
return 0;
}
int32_t RgaImageProcessor::SwapColor(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data) {
// Calculate width aligned to 4 bytes
int aligned_src_width = (src_width + 3) & ~3; // Round up to nearest multiple of 4
// 1. Get or create source buffer
BufferKey src_key{aligned_src_width, src_height, channels};
auto& src_buffer = GetOrCreateBuffer(src_key);
// 2. Get or create destination buffer
BufferKey dst_key{aligned_src_width, src_height, channels};
auto& dst_buffer = GetOrCreateBuffer(dst_key, false);
// 3. Copy source data to RGA buffer, handle padding row by row
for (int y = 0; y < src_height; y++) {
memcpy(static_cast<uint8_t*>(src_buffer.virtual_addr) + y * aligned_src_width * channels, src_data + y * src_width * channels,
src_width * channels);
// Padding area remains zero
}
dma_sync_cpu_to_device(src_buffer.dma_fd);
dma_sync_cpu_to_device(dst_buffer.dma_fd);
// 3. Execute RGA swap color
int ret = imcheck(src_buffer.buffer, dst_buffer.buffer, {}, {});
if (IM_STATUS_NOERROR != ret) {
INSPIRECV_LOG(ERROR) << "RGA parameter check failed: " << imStrError((IM_STATUS)ret);
return -1;
}
// 4. Execute RGA swap color
ret = imcvtcolor(src_buffer.buffer, dst_buffer.buffer, RK_FORMAT_RGB_888, RK_FORMAT_BGR_888);
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA color conversion failed: " << imStrError((IM_STATUS)ret);
return -1;
}
dma_sync_device_to_cpu(dst_buffer.dma_fd);
// 5. Return pointer to destination buffer
*dst_data = static_cast<uint8_t*>(dst_buffer.virtual_addr);
return 0;
}
int32_t RgaImageProcessor::Padding(const uint8_t* src_data, int src_width, int src_height, int channels, int top, int bottom, int left, int right,
uint8_t** dst_data, int& dst_width, int& dst_height) {
// Calculate final dimensions
dst_width = src_width + left + right;
dst_height = src_height + top + bottom;
// Calculate width aligned to 4 bytes
int aligned_src_width = (src_width + 3) & ~3;
int aligned_dst_width = (dst_width + 3) & ~3;
// 1. Get or create source buffer with aligned width
BufferKey src_key{aligned_src_width, src_height, channels};
auto& src_buffer = GetOrCreateBuffer(src_key, true);
// 2. Create destination buffer with padded dimensions
BufferKey dst_key{aligned_dst_width, dst_height, channels};
auto& dst_buffer = GetOrCreateBuffer(dst_key, false);
// 3. Copy source data to RGA buffer
for (int y = 0; y < src_height; y++) {
memcpy(static_cast<uint8_t*>(src_buffer.virtual_addr) + y * aligned_src_width * channels, src_data + y * src_width * channels,
src_width * channels);
}
dma_sync_cpu_to_device(src_buffer.dma_fd);
dma_sync_cpu_to_device(dst_buffer.dma_fd);
// 4. Execute padding operation
// Set source and destination regions
im_rect src_rect = {0, 0, src_width, src_height};
im_rect dst_rect = {left, top, src_width, src_height}; // Specify padding position
int ret = imcheck(src_buffer.buffer, dst_buffer.buffer, src_rect, dst_rect);
if (IM_STATUS_NOERROR != ret) {
INSPIRECV_LOG(ERROR) << "RGA parameter check failed: " << imStrError((IM_STATUS)ret);
return -1;
}
// 5. Fill entire destination area with black first
ret = imfill(dst_buffer.buffer, {0, 0, dst_width, dst_height}, 0x000000);
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA fill failed: " << imStrError((IM_STATUS)ret);
return -1;
}
// 6. Copy source image to specified position in destination
ret = improcess(src_buffer.buffer, dst_buffer.buffer, {}, src_rect, dst_rect, {}, IM_SYNC);
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA copy failed: " << imStrError((IM_STATUS)ret);
return -1;
}
dma_sync_device_to_cpu(dst_buffer.dma_fd);
*dst_data = static_cast<uint8_t*>(dst_buffer.virtual_addr);
return 0;
}
int32_t RgaImageProcessor::ResizeAndPadding(const uint8_t* src_data, int src_width, int src_height, int channels, int dst_width, int dst_height,
uint8_t** dst_data, float& scale) {
// Ensure target dimensions are multiples of 4
int aligned_dst_width = (dst_width + 3) & ~3;
int aligned_dst_height = (dst_height + 3) & ~3;
// Calculate scale (take minimum to fit target box)
scale = std::min(static_cast<float>(aligned_dst_width) / src_width, static_cast<float>(aligned_dst_height) / src_height);
// Calculate scaled dimensions
int resized_w = static_cast<int>(src_width * scale);
int resized_h = static_cast<int>(src_height * scale);
// Ensure scaled dimensions are multiples of 4
resized_w = (resized_w + 3) & ~3;
resized_h = (resized_h + 3) & ~3;
int aligned_src_width = (src_width + 3) & ~3;
// 1. Get source buffer
BufferKey src_key{aligned_src_width, src_height, channels};
auto& src_buffer = GetOrCreateBuffer(src_key);
// 2. Get destination buffer
BufferKey dst_key{aligned_dst_width, aligned_dst_height, channels};
auto& dst_buffer = GetOrCreateBuffer(dst_key, false);
// 3. Copy source data to RGA buffer
for (int y = 0; y < src_height; y++) {
memcpy(static_cast<uint8_t*>(src_buffer.virtual_addr) + y * aligned_src_width * channels, src_data + y * src_width * channels,
src_width * channels);
}
dma_sync_cpu_to_device(src_buffer.dma_fd);
dma_sync_cpu_to_device(dst_buffer.dma_fd);
// 4. Set source and destination regions
im_rect src_rect = {0, 0, src_width, src_height};
im_rect dst_rect = {0, 0, resized_w, resized_h}; // Image placed in top-left corner
int ret = imcheck(src_buffer.buffer, dst_buffer.buffer, src_rect, dst_rect);
if (IM_STATUS_NOERROR != ret) {
INSPIRECV_LOG(ERROR) << "RGA parameter check failed: " << imStrError((IM_STATUS)ret);
return -1;
}
// 5. Fill entire destination area with black first
im_rect fill_rect = {0, 0, aligned_dst_width, aligned_dst_height};
ret = imfill(dst_buffer.buffer, fill_rect, 0x000000); // Fill with black
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA fill failed: " << imStrError((IM_STATUS)ret);
return -1;
}
// 6. Execute resize operation, image will be placed in top-left corner
ret = improcess(src_buffer.buffer, dst_buffer.buffer, {}, src_rect, dst_rect, {}, IM_SYNC);
if (ret != IM_STATUS_SUCCESS) {
INSPIRECV_LOG(ERROR) << "RGA resize failed: " << imStrError((IM_STATUS)ret);
return -1;
}
dma_sync_device_to_cpu(dst_buffer.dma_fd);
// 7. Return processed data
*dst_data = static_cast<uint8_t*>(dst_buffer.virtual_addr);
return 0;
}
} // namespace nexus
} // namespace inspire
#endif // ISF_ENABLE_RGA

211
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/image_processor_rga.h Normal file
View File

@@ -0,0 +1,211 @@
#ifndef INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_RGA_H
#define INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_RGA_H
#if defined(ISF_ENABLE_RGA)
#include "image_processor.h"
#include <linux/stddef.h>
#include <iostream>
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <time.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <signal.h>
#include <unistd.h>
#include <memory>
#include <unordered_map>
#include "im2d.hpp"
#include "im2d_single.h"
#include "RgaUtils.h"
#include "rga/utils.h"
#include "rga/dma_alloc.h"
#include "initialization_module/launch.h"
namespace inspire {
namespace nexus {
class RgaImageProcessor : public ImageProcessor {
public:
RgaImageProcessor();
~RgaImageProcessor() override;
int32_t Resize(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data, int dst_width, int dst_height) override;
int32_t SwapColor(const uint8_t* src_data, int src_width, int src_height, int channels, uint8_t** dst_data) override;
int32_t Padding(const uint8_t* src_data, int src_width, int src_height, int channels, int top, int bottom, int left, int right,
uint8_t** dst_data, int& dst_width, int& dst_height) override;
int32_t ResizeAndPadding(const uint8_t* src_data, int src_width, int src_height, int channels, int dst_width, int dst_height, uint8_t** dst_data,
float& scale) override;
int32_t MarkDone() override;
public:
struct BufferInfo {
int dma_fd;
int width;
int height;
int channels;
size_t buffer_size;
};
BufferInfo GetCurrentSrcBufferInfo() const {
auto it = buffer_cache_.find(last_src_key_);
if (it != buffer_cache_.end()) {
const auto& buffer = it->second;
return {buffer.dma_fd, buffer.width, buffer.height, buffer.channels, buffer.buffer_size};
}
return {-1, 0, 0, 0, 0}; // Return invalid values if cache doesn't exist
}
BufferInfo GetCurrentDstBufferInfo() const {
auto it = buffer_cache_.find(last_dst_key_);
if (it != buffer_cache_.end()) {
const auto& buffer = it->second;
return {buffer.dma_fd, buffer.width, buffer.height, buffer.channels, buffer.buffer_size};
}
return {-1, 0, 0, 0, 0}; // Return invalid values if cache doesn't exist
}
size_t GetCacheSize() const {
return buffer_cache_.size();
}
void DumpCacheStatus() const override {
INSPIRECV_LOG(INFO) << "Current cache status:";
INSPIRECV_LOG(INFO) << "Cache size: " << buffer_cache_.size();
auto src_info = GetCurrentSrcBufferInfo();
INSPIRECV_LOG(INFO) << "Source buffer: "
<< "dma_fd=" << src_info.dma_fd << ", size=" << src_info.width << "x" << src_info.height << "x" << src_info.channels;
auto dst_info = GetCurrentDstBufferInfo();
INSPIRECV_LOG(INFO) << "Destination buffer: "
<< "dma_fd=" << dst_info.dma_fd << ", size=" << dst_info.width << "x" << dst_info.height << "x" << dst_info.channels;
}
private:
struct RGABuffer {
int width{0};
int height{0};
int channels{0};
int dma_fd{-1};
void* virtual_addr{nullptr};
size_t buffer_size{0};
rga_buffer_handle_t handle{0};
rga_buffer_t buffer{};
bool Allocate(int w, int h, int c) {
width = w;
height = h;
channels = c;
buffer_size = width * height * channels;
int ret = dma_buf_alloc(INSPIRE_LAUNCH->GetRockchipDmaHeapPath().c_str(), buffer_size, &dma_fd, &virtual_addr);
if (ret < 0) {
INSPIRECV_LOG(ERROR) << "Failed to allocate DMA buffer: " << ret;
return false;
}
handle = importbuffer_fd(dma_fd, buffer_size);
if (handle == 0) {
INSPIRECV_LOG(ERROR) << "Failed to import buffer";
Release();
return false;
}
buffer = wrapbuffer_handle(handle, w, h, RK_FORMAT_RGB_888);
return true;
}
void Release() {
if (handle) {
releasebuffer_handle(handle);
handle = 0;
}
if (dma_fd >= 0) {
dma_buf_free(buffer_size, &dma_fd, virtual_addr);
dma_fd = -1;
virtual_addr = nullptr;
}
}
~RGABuffer() {
Release();
}
};
struct BufferKey {
int width;
int height;
int channels;
bool operator==(const BufferKey& other) const {
return width == other.width && height == other.height && channels == other.channels;
}
};
struct BufferKeyHash {
std::size_t operator()(const BufferKey& key) const {
return std::hash<int>()(key.width) ^ (std::hash<int>()(key.height) << 1) ^ (std::hash<int>()(key.channels) << 2);
}
};
RGABuffer& GetOrCreateBuffer(const BufferKey& key, bool is_src = true) {
auto it = buffer_cache_.find(key);
if (it != buffer_cache_.end()) {
if (is_src) {
last_src_key_ = key;
} else {
last_dst_key_ = key;
}
return it->second;
}
if (buffer_cache_.size() >= 3) { // Keep max 3 buffers in cache
for (auto it = buffer_cache_.begin(); it != buffer_cache_.end();) {
if (!(it->first == last_src_key_) && !(it->first == last_dst_key_)) {
it = buffer_cache_.erase(it);
} else {
++it;
}
}
}
auto& buffer = buffer_cache_[key];
if (!buffer.Allocate(key.width, key.height, key.channels)) {
INSPIRECV_LOG(ERROR) << "Failed to allocate RGA buffer";
throw std::runtime_error("RGA buffer allocation failed");
}
if (is_src) {
last_src_key_ = key;
} else {
last_dst_key_ = key;
}
return buffer;
}
private:
std::unordered_map<BufferKey, RGABuffer, BufferKeyHash> buffer_cache_;
BufferKey last_src_key_{0, 0, 0};
BufferKey last_dst_key_{0, 0, 0};
};
} // namespace nexus
} // namespace inspire
#endif // ISF_ENABLE_RGA
#endif // INSPIRE_FACE_NEXUS_IMAGE_PROCESSOR_RGA_H

133
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/rga/dma_alloc.cpp Normal file
View File

@@ -0,0 +1,133 @@
#if defined(ISF_ENABLE_RGA)
#include <getopt.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <time.h>
#include <stdbool.h>
#include <assert.h>
#include <endian.h>
#include <errno.h>
#include <fcntl.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/poll.h>
#include <unistd.h>
#include <stdbool.h>
#include <sys/eventfd.h>
#include <sched.h>
#include <pthread.h>
#include <stdint.h>
#include <math.h>
#include <memory.h>
#include <sys/time.h>
#include "dma_alloc.h"
#include "RgaUtils.h"
typedef unsigned long long __u64;
typedef unsigned int __u32;
struct dma_heap_allocation_data {
__u64 len;
__u32 fd;
__u32 fd_flags;
__u64 heap_flags;
};
#define DMA_HEAP_IOC_MAGIC 'H'
#define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOC_MAGIC, 0x0, struct dma_heap_allocation_data)
#define DMA_BUF_SYNC_READ (1 << 0)
#define DMA_BUF_SYNC_WRITE (2 << 0)
#define DMA_BUF_SYNC_RW (DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE)
#define DMA_BUF_SYNC_START (0 << 2)
#define DMA_BUF_SYNC_END (1 << 2)
struct dma_buf_sync {
__u64 flags;
};
#define DMA_BUF_BASE 'b'
#define DMA_BUF_IOCTL_SYNC _IOW(DMA_BUF_BASE, 0, struct dma_buf_sync)
#define CMA_HEAP_SIZE 1024 * 1024
int dma_sync_device_to_cpu(int fd) {
struct dma_buf_sync sync = {0};
sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_RW;
return ioctl(fd, DMA_BUF_IOCTL_SYNC, &sync);
}
int dma_sync_cpu_to_device(int fd) {
struct dma_buf_sync sync = {0};
sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_RW;
return ioctl(fd, DMA_BUF_IOCTL_SYNC, &sync);
}
int dma_buf_alloc(const char *path, size_t size, int *fd, void **va) {
int ret;
int prot;
void *mmap_va;
int dma_heap_fd = -1;
struct dma_heap_allocation_data buf_data;
/* open dma_heap fd */
dma_heap_fd = open(path, O_RDWR);
if (dma_heap_fd < 0) {
printf("open %s fail!\n", path);
return dma_heap_fd;
}
/* alloc buffer */
memset(&buf_data, 0x0, sizeof(struct dma_heap_allocation_data));
buf_data.len = size;
buf_data.fd_flags = O_CLOEXEC | O_RDWR;
ret = ioctl(dma_heap_fd, DMA_HEAP_IOCTL_ALLOC, &buf_data);
if (ret < 0) {
printf("RK_DMA_HEAP_ALLOC_BUFFER failed\n");
return ret;
}
/* mmap va */
if (fcntl(buf_data.fd, F_GETFL) & O_RDWR)
prot = PROT_READ | PROT_WRITE;
else
prot = PROT_READ;
/* mmap contiguors buffer to user */
mmap_va = (void *)mmap(NULL, buf_data.len, prot, MAP_SHARED, buf_data.fd, 0);
if (mmap_va == MAP_FAILED) {
printf("mmap failed: %s\n", strerror(errno));
return -errno;
}
*va = mmap_va;
*fd = buf_data.fd;
close(dma_heap_fd);
return 0;
}
void dma_buf_free(size_t size, int *fd, void *va) {
int len;
len = size;
munmap(va, len);
close(*fd);
*fd = -1;
}
#endif // ISF_ENABLE_RGA

21
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/rga/dma_alloc.h Normal file
View File

@@ -0,0 +1,21 @@
#ifndef RGA_ALLOCATOR_DMA_ALLOC_H__
#define RGA_ALLOCATOR_DMA_ALLOC_H__
#if defined(ISF_ENABLE_RGA)
#define DMA_HEAP_UNCACHE_PATH "/dev/dma_heap/system-uncached"
#define DMA_HEAP_PATH "/dev/dma_heap/system"
#define DMA_HEAP_DMA32_UNCACHE_PATCH "/dev/dma_heap/system-uncached-dma32"
#define DMA_HEAP_DMA32_PATCH "/dev/dma_heap/system-dma32"
#define CMA_HEAP_UNCACHE_PATH "/dev/dma_heap/cma-uncached"
#define RV1106_CMA_HEAP_PATH "/dev/rk_dma_heap/rk-dma-heap-cma"
int dma_sync_device_to_cpu(int fd);
int dma_sync_cpu_to_device(int fd);
int dma_buf_alloc(const char *path, size_t size, int *fd, void **va);
void dma_buf_free(size_t size, int *fd, void *va);
#endif /* #ifndef RGA_ALLOCATOR_DMA_ALLOC_H__ */
#endif // ISF_ENABLE_RGA

183
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/rga/utils.cpp Normal file
View File

@@ -0,0 +1,183 @@
#if defined(ISF_ENABLE_RGA)
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstddef>
#include <cmath>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
#include "RgaUtils.h"
int64_t get_cur_us() {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec * 1000000 + tv.tv_usec;
}
int64_t get_cur_ms() {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec * 1000 + tv.tv_usec / 1000;
}
void draw_rgba(char *buffer, int width, int height) {
for (int i = 0; i < height; i++) {
for (int j = 0; j < width / 4; j++) {
buffer[(i * width * 4) + j * 4 + 0] = 0xff; // R
buffer[(i * width * 4) + j * 4 + 1] = 0x00; // G
buffer[(i * width * 4) + j * 4 + 2] = 0x00; // B
buffer[(i * width * 4) + j * 4 + 3] = 0xff; // A
}
for (int j = width / 4; j < width / 4 * 2; j++) {
buffer[(i * width * 4) + j * 4 + 0] = 0x00;
buffer[(i * width * 4) + j * 4 + 1] = 0xff;
buffer[(i * width * 4) + j * 4 + 2] = 0x00;
buffer[(i * width * 4) + j * 4 + 3] = 0xff;
}
for (int j = width / 4 * 2; j < width / 4 * 3; j++) {
buffer[(i * width * 4) + j * 4 + 0] = 0x00;
buffer[(i * width * 4) + j * 4 + 1] = 0x00;
buffer[(i * width * 4) + j * 4 + 2] = 0xff;
buffer[(i * width * 4) + j * 4 + 3] = 0xff;
}
for (int j = width / 4 * 3; j < width; j++) {
buffer[(i * width * 4) + j * 4 + 0] = 0xff;
buffer[(i * width * 4) + j * 4 + 1] = 0xff;
buffer[(i * width * 4) + j * 4 + 2] = 0xff;
buffer[(i * width * 4) + j * 4 + 3] = 0xff;
}
}
}
void draw_YUV420(char *buffer, int width, int height) {
/* Y channel */
memset(buffer, 0xa8, width * height / 2);
memset(buffer + width * height / 2, 0x54, width * height / 2);
/* UV channel */
memset(buffer + width * height, 0x80, width * height / 4);
memset(buffer + (int)(width * height * 1.25), 0x30, width * height / 4);
}
void draw_YUV422(char *buffer, int width, int height) {
/* Y channel */
memset(buffer, 0xa8, width * height / 2);
memset(buffer + width * height / 2, 0x54, width * height / 2);
/* UV channel */
memset(buffer + width * height, 0x80, width * height / 2);
memset(buffer + (int)(width * height * 1.5), 0x30, width * height / 2);
}
void draw_gray256(char *buffer, int width, int height) {
for (int i = 0; i < height; i++) {
for (int j = 0; j < width / 4; j++) {
buffer[(i * width * 4) + j * 4] = 0xa8;
}
for (int j = width / 4; j < width / 4 * 2; j++) {
buffer[(i * width * 4) + j * 4] = 0x80;
}
for (int j = width / 4 * 2; j < width / 4 * 3; j++) {
buffer[(i * width * 4) + j * 4] = 0x54;
}
for (int j = width / 4 * 3; j < width; j++) {
buffer[(i * width * 4) + j * 4] = 0x30;
}
}
}
int read_image_from_fbc_file(void *buf, const char *path, int sw, int sh, int fmt, int index) {
int size;
char filePath[100];
const char *inputFbcFilePath = "%s/in%dw%d-h%d-%s-fbc.bin";
snprintf(filePath, 100, inputFbcFilePath, path, index, sw, sh, translate_format_str(fmt));
FILE *file = fopen(filePath, "rb");
if (!file) {
fprintf(stderr, "Could not open %s\n", filePath);
return -EINVAL;
}
size = sw * sh * get_bpp_from_format(fmt) * 1.5;
fread(buf, size, 1, file);
fclose(file);
return 0;
}
int read_image_from_file(void *buf, const char *path, int sw, int sh, int fmt, int index) {
int size;
char filePath[100];
const char *inputFilePath = "%s/in%dw%d-h%d-%s.bin";
snprintf(filePath, 100, inputFilePath, path, index, sw, sh, translate_format_str(fmt));
FILE *file = fopen(filePath, "rb");
if (!file) {
fprintf(stderr, "Could not open %s\n", filePath);
return -EINVAL;
}
size = sw * sh * get_bpp_from_format(fmt);
fread(buf, size, 1, file);
fclose(file);
return 0;
}
int write_image_to_fbc_file(void *buf, const char *path, int sw, int sh, int fmt, int index) {
int size;
char filePath[100];
const char *outputFbcFilePath = "%s/out%dw%d-h%d-%s-fbc.bin";
snprintf(filePath, 100, outputFbcFilePath, path, index, sw, sh, translate_format_str(fmt));
FILE *file = fopen(filePath, "wb+");
if (!file) {
fprintf(stderr, "Could not open %s\n", filePath);
return false;
} else {
fprintf(stderr, "open %s and write ok\n", filePath);
}
size = sw * sh * get_bpp_from_format(fmt) * 1.5;
fwrite(buf, size, 1, file);
fclose(file);
return 0;
}
int write_image_to_file(void *buf, const char *path, int sw, int sh, int fmt, int index) {
int size;
char filePath[100];
const char *outputFilePath = "%s/out%dw%d-h%d-%s.bin";
snprintf(filePath, 100, outputFilePath, path, index, sw, sh, translate_format_str(fmt));
FILE *file = fopen(filePath, "wb+");
if (!file) {
fprintf(stderr, "Could not open %s\n", filePath);
return false;
} else {
fprintf(stderr, "open %s and write ok\n", filePath);
}
size = sw * sh * get_bpp_from_format(fmt);
fwrite(buf, size, 1, file);
fclose(file);
return 0;
}
#endif // ISF_ENABLE_RGA

21
cpp-package/inspireface/cpp/inspireface/middleware/nexus_processor/rga/utils.h Normal file
View File

@@ -0,0 +1,21 @@
#ifndef RGA_UTILS_H__
#define RGA_UTILS_H__
#include <stdlib.h>
#if defined(ISF_ENABLE_RGA)
int64_t get_cur_us();
int64_t get_cur_ms();
void draw_rgba(char *buffer, int width, int height);
void draw_YUV420(char *buffer, int width, int height);
void draw_YUV422(char *buffer, int width, int height);
void draw_gray256(char *buffer, int width, int height);
int read_image_from_fbc_file(void *buf, const char *path, int sw, int sh, int fmt, int index);
int read_image_from_file(void *buf, const char *path, int sw, int sh, int fmt, int index);
int write_image_to_fbc_file(void *buf, const char *path, int sw, int sh, int fmt, int index);
int write_image_to_file(void *buf, const char *path, int sw, int sh, int fmt, int index);
#endif // ISF_ENABLE_RGA
#endif /* #ifndef RGA_UTILS_H__ */

141
cpp-package/inspireface/cpp/inspireface/middleware/system.h Normal file
View File

@@ -0,0 +1,141 @@
#ifndef INSPIRE_FACE_SYSTEM_H
#define INSPIRE_FACE_SYSTEM_H
#include <string>
#include <utility>
#include <vector>
#include <memory>
#include <string>
#include <initializer_list>
#if defined(_WIN32)
#include <windows.h>
#else
#include <dirent.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
namespace inspire {
namespace os {
template <typename... Args>
inline std::string PathJoin(Args... args) {
std::string result;
#if defined(_WIN32)
const char sep = '\\';
#else
const char sep = '/';
#endif
std::vector<std::string> paths = {args...};
bool first = true;
for (const auto& p : paths) {
if (p.empty())
continue;
if (first) {
result = p;
first = false;
continue;
}
if (result.back() != sep)
result += sep;
result += p;
}
return result;
}
inline std::pair<std::string, std::string> PathSplit(const std::string& path) {
std::string directory, filename;
size_t pos = path.rfind('/');
#ifdef _WIN32
size_t backslash_pos = path.rfind('\\');
if ((pos == std::string::npos) || (backslash_pos != std::string::npos && backslash_pos > pos)) {
pos = backslash_pos;
}
#endif
if (pos == std::string::npos) {
directory = "";
filename = path;
} else if (pos == 0) {
directory = path.substr(0, 1);
filename = path.substr(1);
} else {
directory = path.substr(0, pos);
filename = path.substr(pos + 1);
}
return {directory, filename};
}
inline std::pair<std::string, std::string> SplitExt(const std::string& path) {
std::string basename, extension;
size_t pos = path.rfind('.');
if (pos == std::string::npos) {
basename = path;
extension = "";
} else {
basename = path.substr(0, pos);
extension = path.substr(pos);
}
return {basename, extension};
}
inline std::string Dirname(const std::string& path) {
return PathSplit(path).first;
}
inline std::string Basename(const std::string& path) {
return PathSplit(path).second;
}
#if defined(_WIN32)
inline std::wstring Utf8ToWideChar(const std::string& utf8str) {
int size_required = MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), utf8str.size(), NULL, 0);
std::wstring ws_translated_str(size_required, 0);
MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), utf8str.size(), &ws_translated_str[0], size_required);
return ws_translated_str;
}
#endif
inline bool IsExists(const std::string& path) {
#if defined(_WIN32)
std::wstring ws_path = Utf8ToWideChar(path);
return _waccess(ws_path.c_str(), 0) == 0;
#else
return access(path.c_str(), F_OK) == 0;
#endif
}
inline bool IsDir(const std::string& path) {
#if defined(_WIN32)
return GetFileAttributesA(path.c_str()) == FILE_ATTRIBUTE_DIRECTORY;
#else
struct stat s;
if (stat(path.c_str(), &s) != 0)
return false;
return S_ISDIR(s.st_mode);
#endif
}
inline bool IsFile(const std::string& path) {
#if defined(_WIN32)
return GetFileAttributesA(path.c_str()) == FILE_ATTRIBUTE_ARCHIVE;
#else
struct stat s;
if (stat(path.c_str(), &s) != 0)
return false;
return S_ISREG(s.st_mode);
#endif
}
} // namespace os
} // namespace inspire
#endif // INSPIRE_FACE_SYSTEM_H

148
cpp-package/inspireface/cpp/inspireface/middleware/thread/resource_pool.h Normal file
View File

@@ -0,0 +1,148 @@
#ifndef INSPIRE_RESOURCE_POOL_H
#define INSPIRE_RESOURCE_POOL_H
#include <iostream>
#include <mutex>
#include <queue>
#include <condition_variable>
#include <memory>
#include <functional>
namespace inspire {
namespace parallel {
/**
* @brief ResourcePool is a thread-safe resource pool that can be used to manage resources in a multi-threaded environment.
* @tparam Resource The type of the resource to be managed.
*/
template <typename Resource>
class ResourcePool {
public:
using ResourceDeleter = std::function<void(Resource&)>;
class ResourceGuard {
public:
ResourceGuard(Resource& resource, ResourcePool& pool) : m_resource(resource), m_pool(pool), m_valid(true) {}
// Move constructor
ResourceGuard(ResourceGuard&& other) noexcept : m_resource(other.m_resource), m_pool(other.m_pool), m_valid(other.m_valid) {
other.m_valid = false;
}
// Disable copy
ResourceGuard(const ResourceGuard&) = delete;
ResourceGuard& operator=(const ResourceGuard&) = delete;
~ResourceGuard() {
if (m_valid) {
m_pool.ReturnResource(std::move(m_resource));
}
}
Resource* operator->() {
return &m_resource;
}
Resource& operator*() {
return m_resource;
}
private:
Resource& m_resource;
ResourcePool& m_pool;
bool m_valid;
};
explicit ResourcePool(size_t size, ResourceDeleter deleter = nullptr) : m_deleter(deleter) {
m_resources.reserve(size);
}
~ResourcePool() {
if (m_deleter) {
std::lock_guard<std::mutex> lock(m_mutex);
for (auto& resource : m_resources) {
m_deleter(resource);
}
}
}
void AddResource(Resource&& resource) {
std::lock_guard<std::mutex> lock(m_mutex);
m_resources.push_back(std::move(resource));
m_available_resources.push(&m_resources.back());
m_cv.notify_one();
}
// Acquire resource (blocking mode)
ResourceGuard AcquireResource() {
std::unique_lock<std::mutex> lock(m_mutex);
m_cv.wait(lock, [this] { return !m_available_resources.empty(); });
Resource* resource = m_available_resources.front();
m_available_resources.pop();
return ResourceGuard(*resource, *this);
}
// Try to acquire resource (non-blocking mode), returns nullptr if no resource available
std::unique_ptr<ResourceGuard> TryAcquireResource() {
std::lock_guard<std::mutex> lock(m_mutex);
if (m_available_resources.empty()) {
return nullptr;
}
Resource* resource = m_available_resources.front();
m_available_resources.pop();
return std::unique_ptr<ResourceGuard>(new ResourceGuard(*resource, *this));
}
// Acquire resource with timeout, returns nullptr if timeout
std::unique_ptr<ResourceGuard> AcquireResource(std::chrono::milliseconds timeout) {
std::unique_lock<std::mutex> lock(m_mutex);
if (!m_cv.wait_for(lock, timeout, [this] { return !m_available_resources.empty(); })) {
return nullptr;
}
Resource* resource = m_available_resources.front();
m_available_resources.pop();
return std::unique_ptr<ResourceGuard>(new ResourceGuard(*resource, *this));
}
size_t AvailableCount() const {
std::lock_guard<std::mutex> lock(m_mutex);
return m_available_resources.size();
}
size_t TotalCount() const {
std::lock_guard<std::mutex> lock(m_mutex);
return m_resources.size();
}
private:
void ReturnResource(Resource&& resource) {
std::lock_guard<std::mutex> lock(m_mutex);
for (auto& stored_resource : m_resources) {
if (&stored_resource == &resource) {
m_available_resources.push(&stored_resource);
m_cv.notify_one();
break;
}
}
}
private:
mutable std::mutex m_mutex;
std::condition_variable m_cv;
std::vector<Resource> m_resources; // Store actual resources
std::queue<Resource*> m_available_resources; // Queue of available resources
ResourceDeleter m_deleter; // Resource cleanup callback
friend class ResourceGuard;
};
} // namespace parallel
} // namespace inspire
#endif // INSPIRE_RESOURCE_POOL_H

630
cpp-package/inspireface/cpp/inspireface/middleware/utils.h
View File

@@ -1,8 +1,7 @@
#ifndef TRACKING_LIB_UTILS_H
#define TRACKING_LIB_UTILS_H
#ifndef INSPIRE_FACE_UTILS_H
#define INSPIRE_FACE_UTILS_H
//#include "face_attribute.h"
#include <cmath>
#include <iostream>
#include <string>
@@ -15,8 +14,11 @@
#include <unistd.h>
#endif
#include <inspirecv/inspirecv.h>
namespace inspire {
inline bool IsDirectory(const std::string& path) {
#ifdef _WIN32
DWORD dwAttrib = GetFileAttributes(path.c_str());
@@ -32,557 +34,57 @@ inline bool IsDirectory(const std::string& path) {
}
inline void EstimateHeadPose(const std::vector<cv::Point2f> &current_shape,
cv::Vec3f &eav) {
// head pose estimation by linear regression.
static int HeadPosePointIndexs[] = {94, 59, 27, 20, 69, 45, 50};
int *estimateHeadPosePointIndexs = HeadPosePointIndexs;
static float estimateHeadPose2dArray[] = {
0.139791, 27.4028, 7.02636, -2.48207, 9.59384, 6.03758, 1.27402,
10.4795, 6.20801, 1.17406, 29.1886, 1.67768, 0.306761, -103.832,
5.66238, 4.78663, 17.8726, -15.3623, -5.20016, 9.29488, -11.2495,
-25.1704, 10.8649, -29.4877, -5.62572, 9.0871, -12.0982, -5.19707,
-8.25251, 13.3965, -23.6643, -13.1348, 29.4322, 67.239, 0.666896,
1.84304, -2.83223, 4.56333, -15.885, -4.74948, -3.79454, 12.7986,
-16.1, 1.47175, 4.03941};
cv::Mat estimateHeadPoseMat =
cv::Mat(15, 3, CV_32FC1, estimateHeadPose2dArray);
if (current_shape.empty())
return;
static const int samplePdim = 7;
float miny = 10000000000.0f;
float maxy = 0.0f;
float sumx = 0.0f;
float sumy = 0.0f;
for (int i = 0; i < samplePdim; i++) {
sumx += current_shape[i].x;
float y = current_shape[i].y;
sumy += y;
if (miny > y)
miny = y;
if (maxy < y)
maxy = y;
}
float dist = maxy - miny;
sumx = sumx / samplePdim;
sumy = sumy / samplePdim;
static cv::Mat tmp(1, 2 * samplePdim + 1, CV_32FC1);
for (int i = 0; i < samplePdim; i++) {
tmp.at<float>(i) =
(current_shape[estimateHeadPosePointIndexs[i]].x - sumx) / dist;
tmp.at<float>(i + samplePdim) =
(current_shape[estimateHeadPosePointIndexs[i]].y - sumy) / dist;
}
tmp.at<float>(2 * samplePdim) = 1.0f;
cv::Mat predict = tmp * estimateHeadPoseMat;
eav[0] = predict.at<float>(0);
eav[1] = predict.at<float>(1);
eav[2] = predict.at<float>(2);
}
inline void MinRect(const std::vector<cv::Point2f> &landmarks, int length,
float *rect) {
rect[0] = landmarks[0].x;
rect[1] = landmarks[0].y;
rect[2] = landmarks[0].x;
rect[3] = landmarks[0].y;
for (int i = 0; i < length; i++) {
if (rect[0] > landmarks[i].x)
rect[0] = landmarks[i].x;
else if (rect[2] < landmarks[i].x)
rect[2] = landmarks[i].x;
if (rect[1] > landmarks[i].y)
rect[1] = landmarks[i].y;
else if (rect[3] < landmarks[i].y)
rect[3] = landmarks[i].y;
}
}
inline float PointDistance(const cv::Point2f &a, const cv::Point2f &b) {
float norm = (a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y);
return sqrt(norm);
}
inline cv::Point2f MeanPoint(const std::vector<cv::Point2f> &points) {
assert(points.size() > 0);
cv::Point2f mean;
for (const auto &p: points)
mean += p;
mean /= static_cast<int>(points.size());
return mean;
}
inline void BestFitRect(const std::vector<cv::Point2f> &pre_landmarks, int size,
std::vector<cv::Point2f> &src_fit) {
src_fit.resize(pre_landmarks.size());
std::vector<float> mean_shape_box = {56, 56, 92, 102};
float rect[4];
MinRect(pre_landmarks, size, rect);
float points_height = rect[3] - rect[1];
float points_centerx = (rect[0] + rect[2]) / 2;
float points_centery = (rect[1] + rect[3]) / 2;
float meanshape_centerx = mean_shape_box[0];
float meanshape_centery = mean_shape_box[1];
float scaleHeight = mean_shape_box[3] / points_height;
float scale = scaleHeight;
for (int i = 0; i < size; i++) {
src_fit[i].x =
pre_landmarks[i].x * scale - points_centerx * scale + meanshape_centerx;
src_fit[i].y =
pre_landmarks[i].y * scale - points_centery * scale + meanshape_centery;
}
}
inline void
SimilarityTransformEstimate(const std::vector<cv::Point2f> &src_points,
const std::vector<cv::Point2f> &dst_points,
cv::Mat &matrix) {
assert(matrix.rows == 2);
assert(matrix.cols == 3);
// matrix.create(2,3,CV_64F);
assert(src_points.size() == dst_points.size());
cv::Point2f src_mean = MeanPoint(src_points);
cv::Point2f dst_mean = MeanPoint(dst_points);
// matrix.resize(6);
float src_norm2 = 0.f;
float sum_a = 0.f;
float sum_b = 0.f;
for (int i = 0; i < src_points.size(); i++) {
cv::Point2f src_d = src_points[i] - src_mean;
cv::Point2f dst_d = dst_points[i] - dst_mean;
src_norm2 += src_d.x * src_d.x + src_d.y * src_d.y;
sum_a += src_d.x * dst_d.x + src_d.y * dst_d.y;
sum_b += src_d.x * dst_d.y - src_d.y * dst_d.x;
}
if (std::fabs(src_norm2) < std::numeric_limits<float>::epsilon()) {
float a = 1.f;
float b = 0.f;
float tx = dst_mean.x - src_mean.x;
float ty = dst_mean.y - src_mean.y;
matrix.at<double>(0, 0) = a;
matrix.at<double>(0, 1) = -b;
matrix.at<double>(0, 2) = tx;
matrix.at<double>(1, 0) = b;
matrix.at<double>(1, 1) = a;
matrix.at<double>(1, 2) = ty;
} else {
float a = sum_a / src_norm2;
float b = sum_b / src_norm2;
float tx = dst_mean.x - (a * src_mean.x - b * src_mean.y);
float ty = dst_mean.y - (b * src_mean.x + a * src_mean.y);
matrix.at<double>(0, 0) = a;
matrix.at<double>(0, 1) = -b;
matrix.at<double>(0, 2) = tx;
matrix.at<double>(1, 0) = b;
matrix.at<double>(1, 1) = a;
matrix.at<double>(1, 2) = ty;
}
}
inline void
SimilarityTransformEstimate(const std::vector<cv::Point2f> &src_points,
const std::vector<cv::Point2f> &dst_points,
std::vector<float> &matrix) {
assert(src_points.size() == dst_points.size());
cv::Point2f src_mean = MeanPoint(src_points);
cv::Point2f dst_mean = MeanPoint(dst_points);
matrix.resize(6);
float src_norm2 = 0.f;
float sum_a = 0.f;
float sum_b = 0.f;
for (int i = 0; i < src_points.size(); i++) {
cv::Point2f src_d = src_points[i] - src_mean;
cv::Point2f dst_d = dst_points[i] - dst_mean;
src_norm2 += src_d.x * src_d.x + src_d.y * src_d.y;
sum_a += src_d.x * dst_d.x + src_d.y * dst_d.y;
sum_b += src_d.x * dst_d.y - src_d.y * dst_d.x;
}
if (std::fabs(src_norm2) < std::numeric_limits<float>::epsilon()) {
float a = 1.f;
float b = 0.f;
float tx = dst_mean.x - src_mean.x;
float ty = dst_mean.y - src_mean.y;
matrix[0] = a;
matrix[1] = -b;
matrix[2] = tx;
matrix[3] = b;
matrix[4] = a;
matrix[5] = ty;
} else {
float a = sum_a / src_norm2;
float b = sum_b / src_norm2;
float tx = dst_mean.x - (a * src_mean.x - b * src_mean.y);
float ty = dst_mean.y - (b * src_mean.x + a * src_mean.y);
matrix[0] = a;
matrix[1] = -b;
matrix[2] = tx;
matrix[3] = b;
matrix[4] = a;
matrix[5] = ty;
}
}
inline cv::Mat GetRectSquareAffine(cv::Rect rect, float win_size = 112) {
assert(rect.height == rect.width);
std::vector<cv::Point2f> dst_pts = {
{0, 0},
{win_size, 0},
{win_size, win_size}};
float x1 = static_cast<float>(rect.x);
float y1 = static_cast<float>(rect.y);
float x2 = static_cast<float>(rect.x + rect.width);
float y2 = static_cast<float>(rect.y + rect.height);
std::vector<cv::Point2f> src_pts = {{x1, y1},
{x2, y1},
{x2, y2}};
cv::Mat m = cv::getAffineTransform(src_pts, dst_pts);
return m;
}
inline cv::Mat SquareToSquare(cv::Rect src, cv::Rect dst,
inline inspirecv::TransformMatrix SquareToSquare(inspirecv::Rect2f src, inspirecv::Rect2f dst,
float win_size = 112) {
float src_x1 = static_cast<float>(src.x);
float src_y1 = static_cast<float>(src.y);
float src_x2 = static_cast<float>(src.x + src.width);
float src_y2 = static_cast<float>(src.y + src.height);
float src_x1 = static_cast<float>(src.GetX());
float src_y1 = static_cast<float>(src.GetY());
float src_x2 = static_cast<float>(src.GetX() + src.GetWidth());
float src_y2 = static_cast<float>(src.GetY() + src.GetHeight());
float dst_x1 = static_cast<float>(dst.x);
float dst_y1 = static_cast<float>(dst.y);
float dst_x2 = static_cast<float>(dst.x + dst.width);
float dst_y2 = static_cast<float>(dst.y + dst.height);
float dst_x1 = static_cast<float>(dst.GetX());
float dst_y1 = static_cast<float>(dst.GetY());
float dst_x2 = static_cast<float>(dst.GetX() + dst.GetWidth());
float dst_y2 = static_cast<float>(dst.GetY() + dst.GetHeight());
std::vector<cv::Point2f> src_pts = {
std::vector<inspirecv::Point2f> src_pts = {
{src_x1, src_y1},
{src_x2, src_y1},
{src_x2, src_y2}};
std::vector<cv::Point2f> dst_pts = {
{src_x2, src_y2},
{src_x1, src_y2}};
std::vector<inspirecv::Point2f> dst_pts = {
{dst_x1, dst_y1},
{dst_x2, dst_y1},
{dst_x2, dst_y2}};
cv::Mat m = cv::getAffineTransform(src_pts, dst_pts);
{dst_x2, dst_y2},
{dst_x1, dst_y2}};
inspirecv::TransformMatrix m = inspirecv::SimilarityTransformEstimate(src_pts, dst_pts);
return m;
}
inline std::vector<cv::Point2f>
ApplyTransformToPoints(const std::vector<cv::Point2f> &points,
const cv::Mat &matrix) {
assert(matrix.rows == 2);
assert(matrix.cols == 3);
double m00 = matrix.at<double>(0, 0);
double m01 = matrix.at<double>(0, 1);
double m02 = matrix.at<double>(0, 2);
double m10 = matrix.at<double>(1, 0);
double m11 = matrix.at<double>(1, 1);
double m12 = matrix.at<double>(1, 2);
std::vector<cv::Point2f> out_points(points.size());
assert(out_points.size() == points.size());
for (int j = 0; j < points.size(); j++) {
out_points[j].x = points[j].x * m00 + points[j].y * m01 + m02;
out_points[j].y = points[j].x * m10 + points[j].y * m11 + m12;
}
return out_points;
}
inline std::vector<inspirecv::Point2f>
FixPointsMeanshape(std::vector<inspirecv::Point2f> &points,
const std::vector<inspirecv::Point2f> &mean_shape) {
inline std::vector<cv::Point2f>
FixPointsMeanshape(std::vector<cv::Point2f> &points,
const std::vector<cv::Point2f> &mean_shape) {
inspirecv::Rect2f bbox = inspirecv::MinBoundingRect(points);
int R = std::max(bbox.GetHeight(), bbox.GetWidth());
int cx = bbox.GetX() + bbox.GetWidth() / 2;
int cy = bbox.GetY() + bbox.GetHeight() / 2;
inspirecv::Rect2f old(cx - R / 2, cy - R / 2, R, R);
cv::Rect bbox = cv::boundingRect(points);
int R = std::max(bbox.height, bbox.width);
int cx = bbox.x + bbox.width / 2;
int cy = bbox.y + bbox.height / 2;
cv::Rect old(cx - R / 2, cy - R / 2, R, R);
cv::Rect mean_shape_box = cv::boundingRect(mean_shape);
int m_R = std::max(mean_shape_box.height, mean_shape_box.width);
int m_cx = mean_shape_box.x + mean_shape_box.width / 2;
int m_cy = mean_shape_box.y + mean_shape_box.height / 2;
cv::Rect _new(m_cx - m_R / 2, m_cy - m_R / 2, m_R, m_R);
cv::Mat affine = SquareToSquare(old, _new);
std::vector<cv::Point2f> new_pts = ApplyTransformToPoints(points, affine);
inspirecv::Rect2f mean_shape_box = inspirecv::MinBoundingRect(mean_shape);
int m_R = std::max(mean_shape_box.GetHeight(), mean_shape_box.GetWidth());
int m_cx = mean_shape_box.GetX() + mean_shape_box.GetWidth() / 2;
int m_cy = mean_shape_box.GetY() + mean_shape_box.GetHeight() / 2;
inspirecv::Rect2f _new(m_cx - m_R / 2, m_cy - m_R / 2, m_R, m_R);
inspirecv::TransformMatrix affine = SquareToSquare(old, _new);
std::vector<inspirecv::Point2f> new_pts = ApplyTransformToPoints(points, affine);
return new_pts;
}
inline std::vector<cv::Point2f> FixPoints(std::vector<cv::Point2f> &points) {
// RotPoints(points, -2);
cv::Rect bbox = cv::boundingRect(points);
int R = std::max(bbox.height, bbox.width);
int cx = bbox.x + bbox.width / 2;
int cy = bbox.y + bbox.height / 2;
cv::Rect old(cx - R / 2, cy - R / 2, R, R);
int margin = 0;
int zx = 0;
int zy = 5;
int new_x1 = zx + margin;
int new_y1 = zy + margin;
int new_x2 = 112 + zx - margin;
int new_y2 = 112 + zy - margin;
cv::Rect _new(new_x1, new_y1, new_x2 - new_x1, new_y2 - new_y1);
cv::Mat affine = SquareToSquare(old, _new);
std::vector<cv::Point2f> new_pts = ApplyTransformToPoints(points, affine);
return new_pts;
}
inline cv::Rect ComputeSafeRect(const cv::Rect &region, int height, int width) {
int x1 = region.x;
int y1 = region.y;
int x2 = region.x + region.width;
int y2 = region.y + region.height;
x1 = std::max(0, x1);
y1 = std::max(0, y1);
x2 = std::min(x2, width - 1);
y2 = std::min(y2, height - 1);
cv::Rect safe_rect(x1, y1, x2 - x1, y2 - y1);
return safe_rect;
}
inline void Transform(const std::vector<cv::Point2f> &pre_landmarks,
float *src_fit, const float *meanshape, const int size,
std::vector<float> &rotation,
std::vector<float> &rotation_inv) {
std::vector<float> src(size * 2);
std::vector<float> dst(size * 2);
float src_mean_x = 0, src_mean_y = 0, dst_mean_x = 0, dst_mean_y = 0;
float tx, ty;
float src_norm = 0, sum_a = 0, sum_b = 0;
for (int i = 0; i < size; i++) {
src_mean_x += pre_landmarks[i].x;
src_mean_y += pre_landmarks[i].y;
dst_mean_x += src_fit[2 * i];
dst_mean_y += meanshape[2 * i + 1];
}
src_mean_x = src_mean_x / size;
src_mean_y = src_mean_y / size;
dst_mean_x = dst_mean_x / size;
dst_mean_y = dst_mean_y / size * 1.1;
// dst_mean_y = dst_mean_y / size * 1.1;
for (int i = 0; i < size; i++) {
src[2 * i] = pre_landmarks[i].x - src_mean_x;
src[2 * i + 1] = pre_landmarks[i].y - src_mean_y;
src_norm += src[2 * i] * src[2 * i];
src_norm += src[2 * i + 1] * src[2 * i + 1];
dst[2 * i] = meanshape[2 * i] - dst_mean_x;
dst[2 * i + 1] = meanshape[2 * i + 1] - dst_mean_y;
sum_a += src[2 * i] * dst[2 * i] + src[2 * i + 1] * dst[2 * i + 1];
sum_b += src[2 * i] * dst[2 * i + 1] - src[2 * i + 1] * dst[2 * i];
}
sum_a = sum_a / src_norm;
sum_b = sum_b / src_norm;
tx = dst_mean_x - sum_a * src_mean_x + sum_b * src_mean_y;
ty = dst_mean_y - sum_b * src_mean_x - sum_a * src_mean_y;
rotation.clear();
rotation.push_back(sum_a);
rotation.push_back(-sum_b);
rotation.push_back(tx);
rotation.push_back(sum_b);
rotation.push_back(sum_a);
rotation.push_back(ty);
double m[6];
m[0] = rotation[0];
m[1] = rotation[1];
m[2] = rotation[2];
m[3] = rotation[3];
m[4] = rotation[4];
m[5] = rotation[5];
double D = m[0] * m[4] - m[1] * m[3];
D = D != 0 ? 1. / D : 0;
double A11 = m[4] * D, A22 = m[0] * D;
m[0] = A11;
m[1] *= -D;
m[3] *= -D;
m[4] = A22;
double b1 = -m[0] * m[2] - m[1] * m[5];
double b2 = -m[3] * m[2] - m[4] * m[5];
m[2] = b1;
m[5] = b2;
rotation_inv.clear();
rotation_inv.push_back(m[0]);
rotation_inv.push_back(m[1]);
rotation_inv.push_back(m[2]);
rotation_inv.push_back(m[3]);
rotation_inv.push_back(m[4]);
rotation_inv.push_back(m[5]);
}
inline std::vector<cv::Point2f> Rect2Points(const cv::Rect rect) {
float x1 = static_cast<float>(rect.x);
float y1 = static_cast<float>(rect.y);
float x2 = static_cast<float>(rect.x + rect.width);
float y2 = static_cast<float>(rect.y + rect.height);
std::vector<cv::Point2f> src_pts = {{x1, y1},
{x2, y1},
{x2, y2},
{x1, y2}};
return src_pts;
}
inline std::vector<cv::Point2f> Rect2Points(const cv::Rect2f rect) {
float x1 = rect.x;
float y1 = rect.y;
float x2 = rect.x + rect.width;
float y2 = rect.y + rect.height;
std::vector<cv::Point2f> src_pts = {{x1, y1},
{x2, y1},
{x2, y2},
{x1, y2}};
return src_pts;
}
inline cv::Mat ScaleAffineMatrix(const cv::Mat &affine, float scale,
int origin_width, int origin_height,
int new_width, int new_height) {
std::vector<cv::Point2f> origin_pts =
Rect2Points(cv::Rect(0, 0, origin_width, origin_height));
cv::Mat affine_inv;
cv::invertAffineTransform(affine, affine_inv);
std::vector<cv::Point2f> screen_pts =
ApplyTransformToPoints(origin_pts, affine_inv);
cv::Point2f center;
for (auto &one: screen_pts) {
center.x += one.x * 0.25f;
center.y += one.y * 0.25f;
}
screen_pts[0].x = center.x + (screen_pts[0].x - center.x) * scale;
screen_pts[0].y = center.y + (screen_pts[0].y - center.y) * scale;
screen_pts[1].x = center.x + (screen_pts[1].x - center.x) * scale;
screen_pts[1].y = center.y + (screen_pts[1].y - center.y) * scale;
screen_pts[2].x = center.x + (screen_pts[2].x - center.x) * scale;
screen_pts[2].y = center.y + (screen_pts[2].y - center.y) * scale;
screen_pts[3].x = center.x + (screen_pts[3].x - center.x) * scale;
screen_pts[3].y = center.y + (screen_pts[3].y - center.y) * scale;
std::vector<cv::Point2f> new_pts =
Rect2Points(cv::Rect(0, 0, new_width, new_height));
screen_pts.pop_back();
new_pts.pop_back();
cv::Mat m = cv::getAffineTransform(screen_pts, new_pts);
return m;
}
template<class ForwardIterator>
inline size_t argmax(ForwardIterator first, ForwardIterator last) {
return std::distance(first, std::max_element(first, last));
}
inline void RotPoints(std::vector<cv::Point2f> &pts, float angle) {
float angle_rad = angle * 3.1415 / 180;
float m11 = cos(angle_rad);
float m12 = -sin(angle_rad);
float m21 = sin(angle_rad);
float m22 = cos(angle_rad);
for (auto &one: pts) {
one.x = one.x * m11 + one.y * m12;
one.y = one.x * m21 + one.y * m22;
}
}
inline cv::Rect flipRectWidth(const cv::Rect &rect, const cv::Size &size) {
int x1 = rect.x;
int y1 = rect.y;
int x2 = rect.x + rect.width;
int y2 = rect.y + rect.height;
x1 = size.width - (rect.x + rect.width);
x2 = size.width - rect.x;
// __android_log_print(ANDROID_LOG_ERROR, "flip: ", "[[%d, %d], [%d, %d]]",x1, y1, x2, y2);
return cv::Rect(cv::Point2f(x1, y1), cv::Point2f(x2, y2));
}
inline std::vector<cv::Point2f> RotatePoints(const std::vector<cv::Point2f>& points, float degree,
const cv::Size &image_size) {
int width = image_size.width;
int height = image_size.height;
float radians = degree / 180 * CV_PI;
int heightNew = int(width * fabs(sin(radians)) + height * fabs(cos(radians)));
int widthNew = int(height * fabs(sin(radians)) + width * fabs(cos(radians)));
cv::Mat trans(2, 3, CV_32F);
trans = cv::getRotationMatrix2D(cv::Point2f(width / 2, height / 2), degree, 1);
trans.at<double>(0, 2) += (widthNew - width) / 2;
trans.at<double>(1, 2) += (heightNew - height) / 2;
float point3_array[points.size()][3];
for (int i = 0; i < points.size(); ++i) {
point3_array[i][0] = points[i].x;
point3_array[i][1] = points[i].y;
point3_array[i][2] = 1;
}
trans.convertTo(trans, CV_32F);
cv::Mat mat_point3s(points.size(), 3, CV_32F, point3_array);
// std::cout << mat_point3s << std::endl;
cv::Mat trans_result = mat_point3s * trans.t();
return cv::Mat_<cv::Point2f>(trans_result);
}
inline cv::Mat RotateRect(cv::Rect &rect, std::vector<cv::Point2f> &dst,
cv::Rect &trans_rect, float degree, const cv::Size &image_size) {
int width = image_size.width;
int height = image_size.height;
float radians = degree / 180 * CV_PI;
int heightNew = int(width * fabs(sin(radians)) + height * fabs(cos(radians)));
int widthNew = int(height * fabs(sin(radians)) + width * fabs(cos(radians)));
cv::Mat trans(2, 3, CV_32F);
trans = cv::getRotationMatrix2D(cv::Point2f(width / 2, height / 2), degree, 1);
trans.at<double>(0, 2) += (widthNew - width) / 2;
trans.at<double>(1, 2) += (heightNew - height) / 2;
float xmin = rect.x;
float ymin = rect.y;
float xmax = rect.x + rect.width;
float ymax = rect.y + rect.height;
float points[][3] = {{xmin, ymin, 1},
{xmax, ymin, 1},
{xmax, ymax, 1},
{xmin, ymax, 1}};
trans.convertTo(trans, CV_32F);
cv::Mat t_points(4, 3, CV_32F, points);
cv::Mat trans_points = t_points * trans.t();
dst.clear();
for (int i = 0; i < 4; ++i) {
float x = trans_points.at<float>(i, 0);
float y = trans_points.at<float>(i, 1);
dst.emplace_back(x, y);
}
float min_x = std::numeric_limits<float>::max(), max_x = 0;
float min_y = std::numeric_limits<float>::max(), max_y = 0;
for (int i = 0; i < dst.size(); ++i) {
if (dst[i].x < min_x) {
min_x = dst[i].x;
}
if (dst[i].x > max_x) {
max_x = dst[i].x;
}
if (dst[i].y < min_y) {
min_y = dst[i].y;
}
if (dst[i].y > max_y) {
max_y = dst[i].y;
}
}
trans_rect = cv::Rect(cv::Point2f(min_x, min_y), cv::Point2f(max_x, max_y));
// trans_rect = flipRectWidth(trans_rect, cv::Size(widthNew, heightNew));
return trans;
}
// Structure to hold bounding box coordinates
struct BoundingBox {
@@ -592,21 +94,22 @@ struct BoundingBox {
int right_bottom_y;
};
inline cv::Rect GetNewBox(int src_w, int src_h, cv::Rect bbox, float scale) {
inline inspirecv::Rect2i GetNewBox(int src_w, int src_h, inspirecv::Rect2i bbox, float scale) {
// Convert cv::Rect to BoundingBox
BoundingBox box;
box.left_top_x = bbox.x;
box.left_top_y = bbox.y;
box.right_bottom_x = bbox.x + bbox.width;
box.right_bottom_y = bbox.y + bbox.height;
box.left_top_x = bbox.GetX();
box.left_top_y = bbox.GetY();
box.right_bottom_x = bbox.GetX() + bbox.GetWidth();
box.right_bottom_y = bbox.GetY() + bbox.GetHeight();
// Compute new bounding box
scale = std::min({static_cast<float>(src_h - 1) / bbox.height, static_cast<float>(src_w - 1) / bbox.width, scale});
scale = std::min({static_cast<float>(src_h - 1) / bbox.GetHeight(), static_cast<float>(src_w - 1) / bbox.GetWidth(), scale});
float new_width = bbox.width * scale;
float new_height = bbox.height * scale;
float center_x = bbox.width / 2.0f + bbox.x;
float center_y = bbox.height / 2.0f + bbox.y;
float new_width = bbox.GetWidth() * scale;
float new_height = bbox.GetHeight() * scale;
float center_x = bbox.GetWidth() / 2.0f + bbox.GetX();
float center_y = bbox.GetHeight() / 2.0f + bbox.GetY();
float left_top_x = center_x - new_width / 2.0f;
float left_top_y = center_y - new_height / 2.0f;
@@ -634,49 +137,22 @@ inline cv::Rect GetNewBox(int src_w, int src_h, cv::Rect bbox, float scale) {
}
// Convert back to cv::Rect for output
cv::Rect new_bbox(static_cast<int>(left_top_x), static_cast<int>(left_top_y),
inspirecv::Rect2i new_bbox(static_cast<int>(left_top_x), static_cast<int>(left_top_y),
static_cast<int>(right_bottom_x - left_top_x), static_cast<int>(right_bottom_y - left_top_y));
return new_bbox;
}
template<typename T>
inline bool isShortestSideGreaterThan(const cv::Rect_<T>& rect, T value, float scale) {
inline bool isShortestSideGreaterThan(const inspirecv::Rect<T>& rect, T value, float scale) {
// Find the shortest edge
T shortestSide = std::min(rect.width / scale, rect.height / scale);
T shortestSide = std::min(static_cast<float>(rect.GetWidth()) / scale, static_cast<float>(rect.GetHeight()) / scale);
// Determines whether the shortest edge is greater than the given value
return shortestSide > value;
}
/**
* @brief Computes the affine transformation matrix for face cropping.
* @param rect Rectangle representing the face in the image.
* @return cv::Mat The computed affine transformation matrix.
*/
inline cv::Mat ComputeCropMatrix(const cv::Rect2f &rect, int width, int height) {
float x = rect.x;
float y = rect.y;
float w = rect.width;
float h = rect.height;
float cx = x + w / 2;
float cy = y + h / 2;
float length = std::max(w, h) * 1.5 / 2;
float x1 = cx - length;
float y1 = cy - length;
float x2 = cx + length;
float y2 = cy + length;
cv::Rect2f padding_rect(x1, y1, x2 - x1, y2 - y1);
std::vector<cv::Point2f> rect_pts = Rect2Points(padding_rect);
rect_pts.erase(rect_pts.end() - 1);
std::vector<cv::Point2f> dst_pts = {{0, 0}, {(float )width, 0}, {(float )width, (float )height}};
cv::Mat m = cv::getAffineTransform(rect_pts, dst_pts);
return m;
}
// Exponential Moving Average (EMA) filter function
inline float EmaFilter(float currentProb, std::vector<float>& history, int max, float alpha = 0.2f) {
inline float EmaFilter(float currentProb, std::vector<float> &history, int max, float alpha = 0.2f) {
// Add current probability to history
history.push_back(currentProb);
@@ -694,6 +170,6 @@ inline float EmaFilter(float currentProb, std::vector<float>& history, int max,
return ema;
}
} // namespace inspire
} // namespace inspire
#endif
#endif // INSPIRE_FACE_UTILS_H

13
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/all.h
View File

@@ -1,11 +1,12 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef HYPERFACEREPO_ATTRIBUTE_ALL_H
#define HYPERFACEREPO_ATTRIBUTE_ALL_H
#include "mask_predict.h"
#include "face_attribute.h"
#include "mask_predict_adapt.h"
#include "face_attribute_adapt.h"
#endif //HYPERFACEREPO_ATTRIBUTE_ALL_H
#endif // HYPERFACEREPO_ATTRIBUTE_ALL_H

69
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute.h
View File

@@ -1,69 +0,0 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
#pragma once
#ifndef HYPERFACEREPO_GENDERPREDICT_H
#define HYPERFACEREPO_GENDERPREDICT_H
#include "data_type.h"
#include "middleware/any_net.h"
namespace inspire {
/**
* @class FaceAttributePredict
* @brief According to the face image, three classification information of age, gender and race were extracted.
*
* This class inherits from AnyNet and provides methods for performing face attribute prediction.
*/
class INSPIRE_API FaceAttributePredict : public AnyNet {
public:
/**
* @brief Constructor for FaceAttributePredict class.
*/
FaceAttributePredict();
/**
* @brief Exec infer.
*
* @param bgr_affine The BGR affine matrix to perform mask prediction on.
* @return The multi-list attribute prediction result.
*/
std::vector<int> operator()(const Matrix& bgr_affine);
private:
// Define primitive tag
const std::vector<std::string> m_original_labels_ = {
"Black", "East Asian", "Indian", "Latino_Hispanic", "Middle Eastern", "Southeast Asian", "White"
};
// Define simplified labels
const std::vector<std::string> m_simplified_labels_ = {
"Black", "Asian", "Latino/Hispanic", "Middle Eastern", "White"
};
// Define the mapping from the original tag to the simplified tag
const std::unordered_map<std::string, std::string> m_label_map_ = {
{"Black", "Black"},
{"East Asian", "Asian"},
{"Indian", "Asian"},
{"Latino_Hispanic", "Latino/Hispanic"},
{"Middle Eastern", "Middle Eastern"},
{"Southeast Asian", "Asian"},
{"White", "White"}
};
// Define index maps for simplified labels
const std::unordered_map<std::string, int> m_simplified_label_index_ = {
{"Black", 0},
{"Asian", 1},
{"Latino/Hispanic", 2},
{"Middle Eastern", 3},
{"White", 4}
};
};
} // namespace hyper
#endif //HYPERFACEREPO_GENDERPREDICT_H

32
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute.cpp → cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute_adapt.cpp
View File

@@ -1,17 +1,24 @@
//
// Created by Tunm-Air13 on 2023/9/8.
//
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#include "face_attribute.h"
#include "face_attribute_adapt.h"
#include "middleware/utils.h"
namespace inspire {
FaceAttributePredict::FaceAttributePredict(): AnyNet("FaceAttributePredict") {}
FaceAttributePredictAdapt::FaceAttributePredictAdapt() : AnyNetAdapter("FaceAttributePredictAdapt") {}
std::vector<int> FaceAttributePredict::operator()(const Matrix& bgr_affine) {
std::vector<int> FaceAttributePredictAdapt::operator()(const inspirecv::Image &bgr_affine) {
AnyTensorOutputs outputs;
Forward(bgr_affine, outputs);
if (bgr_affine.Width() != INPUT_WIDTH || bgr_affine.Height() != INPUT_HEIGHT) {
auto resized = bgr_affine.Resize(INPUT_WIDTH, INPUT_HEIGHT);
Forward(resized, outputs);
} else {
Forward(bgr_affine, outputs);
}
// cv::imshow("w", bgr_affine);
// cv::waitKey(0);
@@ -19,11 +26,6 @@ std::vector<int> FaceAttributePredict::operator()(const Matrix& bgr_affine) {
std::vector<float> &genderOut = outputs[1].second;
std::vector<float> &ageOut = outputs[2].second;
// for(int i = 0; i < raceOut.size(); i++) {
// std::cout << raceOut[i] << ", ";
// }
// std::cout << std::endl;
auto raceIdx = argmax(raceOut.begin(), raceOut.end());
auto genderIdx = argmax(genderOut.begin(), genderOut.end());
auto ageIdx = argmax(ageOut.begin(), ageOut.end());
@@ -31,11 +33,11 @@ std::vector<int> FaceAttributePredict::operator()(const Matrix& bgr_affine) {
std::string raceLabel = m_original_labels_[raceIdx];
std::string simplifiedLabel = m_label_map_.at(raceLabel);
int simplifiedRaceIdx = m_simplified_label_index_.at(simplifiedLabel);
// std::cout << raceLabel << std::endl;
// std::cout << simplifiedLabel << std::endl;
return {simplifiedRaceIdx, 1 - (int )genderIdx, (int )ageIdx};
return {simplifiedRaceIdx, 1 - (int)genderIdx, (int)ageIdx};
}
} // namespace hyper
} // namespace inspire

61
cpp-package/inspireface/cpp/inspireface/pipeline_module/attribute/face_attribute_adapt.h Normal file
View File

@@ -0,0 +1,61 @@
/**
* Created by Jingyu Yan
* @date 2024-10-01
*/
#pragma once
#ifndef INSPIRE_FACE_ATTRIBUTE_ADAPT_H
#define INSPIRE_FACE_ATTRIBUTE_ADAPT_H
#include "data_type.h"
#include "middleware/any_net_adapter.h"
namespace inspire {
/**
* @class FaceAttributePredict
* @brief According to the face image, three classification information of age, gender and race were extracted.
*
* This class inherits from AnyNet and provides methods for performing face attribute prediction.
*/
class INSPIRE_API FaceAttributePredictAdapt : public AnyNetAdapter {
public:
int32_t INPUT_WIDTH = 112;
int32_t INPUT_HEIGHT = 112;
/**
* @brief Constructor for FaceAttributePredict class.
*/
FaceAttributePredictAdapt();
/**
* @brief Exec infer.
*
* @param bgr_affine The BGR affine matrix to perform mask prediction on.
* @return The multi-list attribute prediction result.
*/
std::vector<int> operator()(const inspirecv::Image& bgr_affine);
private:
// Define primitive tag
const std::vector<std::string> m_original_labels_ = {"Black", "East Asian", "Indian", "Latino_Hispanic",
"Middle Eastern", "Southeast Asian", "White"};
// Define simplified labels
const std::vector<std::string> m_simplified_labels_ = {"Black", "Asian", "Latino/Hispanic", "Middle Eastern", "White"};
// Define the mapping from the original tag to the simplified tag
const std::unordered_map<std::string, std::string> m_label_map_ = {{"Black", "Black"},
{"East Asian", "Asian"},
{"Indian", "Asian"},
{"Latino_Hispanic", "Latino/Hispanic"},
{"Middle Eastern", "Middle Eastern"},
{"Southeast Asian", "Asian"},
{"White", "White"}};
// Define index maps for simplified labels
const std::unordered_map<std::string, int> m_simplified_label_index_ = {
{"Black", 0}, {"Asian", 1}, {"Latino/Hispanic", 2}, {"Middle Eastern", 3}, {"White", 4}};
};
} // namespace inspire
#endif // INSPIRE_FACE_ATTRIBUTE_ADAPT_H

Some files were not shown because too many files have changed in this diff Show More