ZF/SmartJavaAI
1
0
Fork 0
mirror of https://github.com/geekwenjie/SmartJavaAI.git synced 2026-09-13 13:18:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

1、FaceNet 特征提取新增人脸对齐

Browse Source 
2、人脸检测新5点人脸关键点定位
3、特征提取接口支持多人脸和最佳人脸提取
4、修复人脸框边界精度问题
5、更新 Maven 发布的 groupId
This commit is contained in:
dengwenjie
2025-04-28 16:13:34 +08:00
parent 2fdc20f610
commit 42d2943a94
50 changed files with 3413 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
smartjavaai-face/src/main/java/cn/smartjavaai/face/AbstractFaceModel.java
View File

@@ -5,6 +5,7 @@ import cn.smartjavaai.face.entity.FaceResult;
import java.awt.image.BufferedImage;
import java.io.InputStream;
import java.util.List;
/**
* 人脸识别算法
@@ -46,16 +47,6 @@ public abstract class AbstractFaceModel implements FaceModel {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] featureExtraction(String imagePath) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] featureExtraction(InputStream inputStream) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
throw new UnsupportedOperationException("默认不支持该功能");
@@ -111,15 +102,6 @@ public abstract class AbstractFaceModel implements FaceModel {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] featureExtraction(BufferedImage sourceImage) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] featureExtraction(byte[] imageData) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) {
@@ -140,4 +122,64 @@ public abstract class AbstractFaceModel implements FaceModel {
public FaceResult search(byte[] imageData) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(String imagePath) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(byte[] imageData) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(BufferedImage image) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(BufferedImage image, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(String imagePath, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public List<float[]> extractFeatures(byte[] imageData, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(BufferedImage image) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(String imagePath) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(byte[] imageData) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(BufferedImage image, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(String imagePath, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
@Override
public float[] extractTopFaceFeature(byte[] imageData, FaceExtractConfig config) {
throw new UnsupportedOperationException("默认不支持该功能");
}
}

38
smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceExtractConfig.java Normal file
View File

@@ -0,0 +1,38 @@
package cn.smartjavaai.face;
import lombok.Data;
/**
* 人脸特征提取配置
* @author dwj
* @date 2025/4/24
*/
@Data
public class FaceExtractConfig {
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
/**
* 是否对齐人脸
*/
private boolean align = true;
/**
* 人脸检测模型配置
*/
private FaceModelConfig detectModelConfig;
public FaceExtractConfig() {
}
public FaceExtractConfig(boolean cropFace, boolean align, FaceModelConfig detectModelConfig) {
this.cropFace = cropFace;
this.align = align;
this.detectModelConfig = detectModelConfig;
}
}

119
smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceModel.java
View File

@@ -5,6 +5,7 @@ import cn.smartjavaai.face.entity.FaceResult;
import java.awt.image.BufferedImage;
import java.io.InputStream;
import java.util.List;
/**
* 人脸识别算法
@@ -61,34 +62,6 @@ public interface FaceModel {
*/
BufferedImage detectAndDraw(BufferedImage sourceImage);
/**
* 特征提取
* @param imagePath 图片路径
* @return
*/
float[] featureExtraction(String imagePath);
/**
* 特征提取
* @param inputStream 输入流
* @return
*/
float[] featureExtraction(InputStream inputStream);
/**
* 特征提取
* @param sourceImage BufferedImage图片数据
* @return
*/
float[] featureExtraction(BufferedImage sourceImage);
/**
* 特征提取
* @param imageData 图片字节流
* @return
*/
float[] featureExtraction(byte[] imageData);
/**
* 计算相似度
* @param feature1 图1特征
@@ -203,4 +176,94 @@ public interface FaceModel {
*/
long clearFace();
/**
* 特征提取（使用默认配置）
* @param imagePath 图片路径
* @return
*/
List<float[]> extractFeatures(String imagePath);
/**
* 特征提取（使用默认配置）
* @param imageData 图片字节流
* @return
*/
List<float[]> extractFeatures(byte[] imageData);
/**
* 特征提取（使用默认配置）
* @param image BufferedImage
* @return
*/
List<float[]> extractFeatures(BufferedImage image);
/**
* 特征提取（使用自定义配置）
* 强制裁剪操作
* @param image BufferedImage
* @param config
* @return
*/
List<float[]> extractFeatures(BufferedImage image, FaceExtractConfig config);
/**
* 特征提取（使用自定义配置）
* @param imagePath 图片路径
* @param config
* @return
*/
List<float[]> extractFeatures(String imagePath, FaceExtractConfig config);
/**
* 特征提取（使用自定义配置）
* @param imageData 图片字节流
* @param config
* @return
*/
List<float[]> extractFeatures(byte[] imageData, FaceExtractConfig config);
/**
* 提取分数最高人脸特征（使用默认配置）
* @param image BufferedImage
* @return
*/
float[] extractTopFaceFeature(BufferedImage image);
/**
* 提取分数最高人脸特征（使用默认配置）
* @param imagePath 图片路径
* @return
*/
float[] extractTopFaceFeature(String imagePath);
/**
* 提取分数最高人脸特征（使用默认配置）
* @param imageData 图片字节流
* @return
*/
float[] extractTopFaceFeature(byte[] imageData);
/**
* 提取分数最高人脸特征（使用自定义配置）
* @param image BufferedImage
* @return
*/
float[] extractTopFaceFeature(BufferedImage image, FaceExtractConfig config);
/**
* 提取分数最高人脸特征（使用自定义配置）
* @param imagePath 图片路径
* @param config
* @return
*/
float[] extractTopFaceFeature(String imagePath, FaceExtractConfig config);
/**
* 提取分数最高人脸特征（使用自定义配置）
* @param imageData 图片字节流
* @param config
* @return
*/
float[] extractTopFaceFeature(byte[] imageData, FaceExtractConfig config);
}

9
smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceModelConfig.java
View File

@@ -50,6 +50,15 @@ public class FaceModelConfig {
*/
private int gpuId = 0;
public FaceModelConfig() {
}
public FaceModelConfig(FaceModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceModelConfig(FaceModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
}

34
smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/SeetaFace6Factory.java
View File

@@ -1,34 +0,0 @@
package cn.smartjavaai.face.factory;
import ai.djl.inference.Predictor;
import ai.djl.repository.zoo.ZooModel;
import cn.smartjavaai.face.model.SeetaFace6Model;
import com.seetaface.SeetaFace6JNI;
import org.apache.commons.pool2.BasePooledObjectFactory;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.impl.DefaultPooledObject;
/**
* Predictor 工厂类
* @author dwj
* @date 2025/4/8
*/
public class SeetaFace6Factory extends BasePooledObjectFactory<SeetaFace6JNI> {
@Override
public SeetaFace6JNI create() {
return new SeetaFace6JNI();
}
@Override
public PooledObject<SeetaFace6JNI> wrap(SeetaFace6JNI obj) {
return new DefaultPooledObject<>(obj);
}
@Override
public void destroyObject(PooledObject<SeetaFace6JNI> p) {
//p.getObject().dispose(); // 如果需要释放 native 资源
SeetaFace6JNI object = p.getObject();
object = null;
}
}

294
smartjavaai-face/src/main/java/cn/smartjavaai/face/model/FeatureExtractionModel.java
View File

@@ -5,31 +5,41 @@ import ai.djl.MalformedModelException;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.AbstractFaceModel;
import cn.smartjavaai.face.FaceModelConfig;
import cn.smartjavaai.face.*;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.translator.FaceFeatureTranslator;
import cn.smartjavaai.face.utils.FaceAlignUtils;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.OpenCVUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@@ -117,65 +127,6 @@ public class FeatureExtractionModel extends AbstractFaceModel implements AutoClo
}
/**
* 特征提取
* @param imagePath 图片路径
* @return
*/
@Override
public float[] featureExtraction(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
} catch (IOException e) {
throw new FaceException("无效图片", e);
}
return featureExtraction(img);
}
/**
* 特征提取
* @param inputStream 输入流
* @return
*/
@Override
public float[] featureExtraction(InputStream inputStream) {
if(Objects.isNull(inputStream)){
throw new FaceException("图像输入流无效");
}
Image img = null;
try {
img = ImageFactory.getInstance().fromInputStream(inputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return featureExtraction(img);
}
@Override
public float[] featureExtraction(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
Image img = ImageFactory.getInstance().fromImage(sourceImage);
return featureExtraction(img);
}
@Override
public float[] featureExtraction(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return featureExtraction(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("无效图片字节流", e);
}
}
/**
* 计算相似度
* @param feature1 图1特征
@@ -207,34 +158,19 @@ public class FeatureExtractionModel extends AbstractFaceModel implements AutoClo
if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){
throw new FaceException("图像文件不存在");
}
float[] feature1 = featureExtraction(imagePath1);
float[] feature2 = featureExtraction(imagePath2);
float[] feature1 = extractTopFaceFeature(imagePath1);
float[] feature2 = extractTopFaceFeature(imagePath2);
return calculSimilar(feature1, feature2);
}
/**
* 特征比较
* @param inputStream1 图1输入流
* @param inputStream2 图2输入流
* @return
*/
@Override
public float featureComparison(InputStream inputStream1, InputStream inputStream2) {
if(Objects.isNull(inputStream1) || Objects.isNull(inputStream2)){
throw new FaceException("图像输入流无效");
}
float[] feature1 = featureExtraction(inputStream1);
float[] feature2 = featureExtraction(inputStream2);
return calculSimilar(feature1, feature2);
}
@Override
public float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) {
if(!ImageUtils.isImageValid(sourceImage1) || !ImageUtils.isImageValid(sourceImag2)){
throw new FaceException("图像无效");
}
float[] feature1 = featureExtraction(sourceImage1);
float[] feature2 = featureExtraction(sourceImag2);
float[] feature1 = extractTopFaceFeature(sourceImage1);
float[] feature2 = extractTopFaceFeature(sourceImag2);
return calculSimilar(feature1, feature2);
}
@@ -243,11 +179,203 @@ public class FeatureExtractionModel extends AbstractFaceModel implements AutoClo
if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){
throw new FaceException("图像无效");
}
float[] feature1 = featureExtraction(imageData1);
float[] feature2 = featureExtraction(imageData2);
float[] feature1 = extractTopFaceFeature(imageData1);
float[] feature2 = extractTopFaceFeature(imageData2);
return calculSimilar(feature1, feature2);
}
/**
* 获取默认特征提取配置
* @return
*/
private FaceExtractConfig getDefaultConfig() {
FaceExtractConfig config = new FaceExtractConfig();
FaceModelConfig detectModelConfig = new FaceModelConfig();
detectModelConfig.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
config.setDetectModelConfig(detectModelConfig);
return config;
}
@Override
public List<float[]> extractFeatures(String imagePath) {
return extractFeatures(imagePath, getDefaultConfig());
}
@Override
public List<float[]> extractFeatures(byte[] imageData) {
return extractFeatures(imageData, getDefaultConfig());
}
@Override
public List<float[]> extractFeatures(BufferedImage image) {
return extractFeatures(image, getDefaultConfig());
}
@Override
public List<float[]> extractFeatures(BufferedImage image, FaceExtractConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
List<float[]> featureList = new ArrayList<float[]>();
if(Objects.isNull(config.getDetectModelConfig())){
throw new FaceException("config.detectModelConfig为null");
}
FaceModel faceModel = FaceModelFactory.getInstance().getModel(config.getDetectModelConfig());
DetectionResponse detectedResult = faceModel.detect(image);
if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getRectangleList()) || detectedResult.getRectangleList().isEmpty()){
throw new FaceException("未检测到人脸");
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
NDManager manager = NDManager.newBaseManager();
for (DetectionRectangle rectangle : detectedResult.getRectangleList()){
float[] features = null;
//裁剪人脸
Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight());
//人脸对齐
if(config.isAlign()){
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(rectangle.getKeyPoints());
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
/*Mat sourceMat = OpenCVUtils.image2Mat(image);
Mat mat = FaceAlignUtils.warpAffine(sourceMat, affine_matrix);
//OpenCVUtils.mat2Image(mat);
Image alignedImg = ImageFactory.getInstance().fromImage(mat);
features = featureExtraction(alignedImg);*/
Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
features = featureExtraction(alignedImg);
}else{
//不对齐人脸
features = featureExtraction(subImage);
}
if(Objects.nonNull(features)){
featureList.add(features);
}
}
return featureList;
}
@Override
public List<float[]> extractFeatures(String imagePath, FaceExtractConfig config) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractFeatures(image, config);
}
@Override
public List<float[]> extractFeatures(byte[] imageData, FaceExtractConfig config) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return extractFeatures(ImageIO.read(new ByteArrayInputStream(imageData)), config);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public float[] extractTopFaceFeature(BufferedImage image) {
return extractTopFaceFeature(image, getDefaultConfig());
}
@Override
public float[] extractTopFaceFeature(String imagePath) {
return extractTopFaceFeature(imagePath, getDefaultConfig());
}
@Override
public float[] extractTopFaceFeature(byte[] imageData) {
return extractTopFaceFeature(imageData, getDefaultConfig());
}
@Override
public float[] extractTopFaceFeature(BufferedImage image, FaceExtractConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(Objects.isNull(config.getDetectModelConfig())){
throw new FaceException("config.detectModelConfig为null");
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
float[] features = null;
if(config.isCropFace()){
FaceModel faceModel = FaceModelFactory.getInstance().getModel(config.getDetectModelConfig());
DetectionResponse detectedResult = faceModel.detect(image);
if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getRectangleList()) || detectedResult.getRectangleList().isEmpty()){
throw new FaceException("未检测到人脸");
}
//只取第一个人脸
DetectionRectangle rectangle = detectedResult.getRectangleList().get(0);
//裁剪人脸
Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight());
//人脸对齐
if(config.isAlign()){
NDManager manager = NDManager.newBaseManager();
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(rectangle.getKeyPoints());
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
/*Mat sourceMat = OpenCVUtils.image2Mat(image);
Mat mat = FaceAlignUtils.warpAffine(sourceMat, affine_matrix);
OpenCVUtils.mat2Image(mat);
Image alignedImg = ImageFactory.getInstance().fromImage(OpenCVUtils.mat2Image(mat));
features = featureExtraction(alignedImg);*/
Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
features = featureExtraction(alignedImg);
}else{
//不对齐人脸
features = featureExtraction(subImage);
}
}else{
//不裁剪人脸直接提取特征
features = featureExtraction(djlImage);
}
return features;
}
@Override
public float[] extractTopFaceFeature(String imagePath, FaceExtractConfig config) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractTopFaceFeature(image, config);
}
@Override
public float[] extractTopFaceFeature(byte[] imageData, FaceExtractConfig config) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return extractTopFaceFeature(ImageIO.read(new ByteArrayInputStream(imageData)), config);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public void close() {
if (predictorPool != null) {
@@ -255,4 +383,6 @@ public class FeatureExtractionModel extends AbstractFaceModel implements AutoClo
}
}
}

5
smartjavaai-face/src/main/java/cn/smartjavaai/face/model/RetinaFaceModel.java
View File

@@ -19,6 +19,7 @@ import cn.smartjavaai.face.*;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.OpenCVUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
@@ -144,7 +145,7 @@ public class RetinaFaceModel extends AbstractFaceModel implements AutoCloseable{
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
Image img = ImageFactory.getInstance().fromImage(image);
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
}
@@ -186,7 +187,7 @@ public class RetinaFaceModel extends AbstractFaceModel implements AutoCloseable{
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
Image img = ImageFactory.getInstance().fromImage(sourceImage);
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");

310
smartjavaai-face/src/main/java/cn/smartjavaai/face/model/SeetaFace6Model.java
View File

@@ -1,29 +1,23 @@
package cn.smartjavaai.face.model;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.AbstractFaceModel;
import cn.smartjavaai.face.FaceExtractConfig;
import cn.smartjavaai.face.FaceModelConfig;
import cn.smartjavaai.face.dao.FaceDao;
import cn.smartjavaai.face.entity.FaceData;
import cn.smartjavaai.face.entity.FaceResult;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.utils.FaceAlignUtils;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import com.seetaface.NativeLoader;
import com.seetaface.SeetaFace6JNI;
import cn.smartjavaai.face.seetaface.NativeLoader;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
@@ -31,6 +25,7 @@ import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -154,10 +149,19 @@ public class SeetaFace6Model extends AbstractFaceModel {
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector predictor = null;
FaceLandmarker faceLandmarker = null;
try {
predictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaRect[] seetaResult = predictor.Detect(imageData);
return FaceUtils.convertToDetectionResponse(seetaResult, config);
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
seetaPointFSList.add(pointFS);
}
return FaceUtils.convertToDetectionResponse(seetaResult, config, seetaPointFSList);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
@@ -168,6 +172,13 @@ public class SeetaFace6Model extends AbstractFaceModel {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@@ -295,104 +306,6 @@ public class SeetaFace6Model extends AbstractFaceModel {
}
}
@Override
public float[] featureExtraction(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceRecognizer = faceRecognizerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("未检测到人脸");
}
//提取第一个人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
throw new FaceException("人脸特征提取失败");
}
return features;
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public float[] featureExtraction(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return featureExtraction(image);
}
@Override
public float[] featureExtraction(InputStream inputStream) {
if(Objects.isNull(inputStream)){
throw new FaceException("图像输入流无效");
}
BufferedImage image = null;
try {
image = ImageIO.read(inputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return featureExtraction(image);
}
@Override
public float[] featureExtraction(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return featureExtraction(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
@@ -897,4 +810,185 @@ public class SeetaFace6Model extends AbstractFaceModel {
pageNo++;
}
}
@Override
public List<float[]> extractFeatures(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractFeatures(image);
}
@Override
public List<float[]> extractFeatures(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return extractFeatures(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public List<float[]> extractFeatures(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
List<float[]> featureList = new ArrayList<float[]>();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceRecognizer = faceRecognizerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("未检测到人脸");
}
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
throw new FaceException("人脸特征提取失败");
}
featureList.add(features);
}
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
return featureList;
}
@Override
public float[] extractTopFaceFeature(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
float[] features = null;
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceRecognizer = faceRecognizerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("未检测到人脸");
}
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
//提取特征
features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
throw new FaceException("人脸特征提取失败");
}
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
return features;
}
@Override
public float[] extractTopFaceFeature(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractTopFaceFeature(image);
}
@Override
public float[] extractTopFaceFeature(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return extractTopFaceFeature(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
}

8
smartjavaai-face/src/main/java/cn/smartjavaai/face/model/UltraLightFastGenericFaceModel.java
View File

@@ -19,6 +19,7 @@ import cn.smartjavaai.face.*;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.OpenCVUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
@@ -133,7 +134,7 @@ public class UltraLightFastGenericFaceModel extends AbstractFaceModel implements
@Override
public DetectionResponse detect(BufferedImage image) {
Image img = ImageFactory.getInstance().fromImage(image);
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
}
@@ -175,7 +176,7 @@ public class UltraLightFastGenericFaceModel extends AbstractFaceModel implements
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
Image img = ImageFactory.getInstance().fromImage(sourceImage);
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");
@@ -222,6 +223,9 @@ public class UltraLightFastGenericFaceModel extends AbstractFaceModel implements
}
@Override
public void close() {
if (predictorPool != null) {

221
smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java Normal file
View File

@@ -0,0 +1,221 @@
package cn.smartjavaai.face.seetaface;
import cn.hutool.core.io.FileUtil;
import cn.hutool.setting.dialect.Props;
import cn.hutool.system.OsInfo;
import cn.hutool.system.SystemUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.FaceModelConfig;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.util.DllItem;
import com.seeta.sdk.util.LoadNativeCore;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.stream.Collectors;
/**
* 依赖库加载器
* @author dwj
*/
@Slf4j
public class NativeLoader {
private static Path seetaface6NativePath;
private static final String SEETAFACE_LIB_DIR = "seetaface6";
/**
* 定义dll 路径和加载顺序的文件
*/
private static final String PROPERTIES_FILE_NAME = "dll.properties";
public static void loadNativeLibraries(FaceModelConfig config) {
try {
OsInfo osInfo = SystemUtil.getOsInfo();
//检查当前系统是否支持
if(!osInfo.isWindows() && !osInfo.isLinux()){
throw new FaceException("当前系统不支持：" + osInfo.getName());
}
//判断硬件架构是否支持GPU
if(config.getDevice() != null && config.getDevice().equals(DeviceEnum.GPU)){
//GPU仅支持amd64
if(!osInfo.getArch().contains("amd64") && !osInfo.getArch().contains("x86_64")){
throw new FaceException("seetaface6 GPU模型不支持当前arch：" + osInfo.getArch());
}
}
seetaface6NativePath = Paths.get(Config.getCachePath(), SEETAFACE_LIB_DIR);
//创建目录
FileUtil.mkdir(seetaface6NativePath);
log.info("seetaface6依赖库路径: " + seetaface6NativePath.toAbsolutePath().toString());
//拷贝依赖库到缓存目录
List<File> fileList = getLibFiles(osInfo, config.getDevice());
if(fileList != null && !fileList.isEmpty()){
// 加载依赖库文件
fileList.forEach(file -> {
System.load(file.getAbsolutePath());
log.info(String.format("load %s finish", file.getAbsolutePath()));
});
}
} catch (Exception e) {
throw new RuntimeException("Native library loading failed", e);
}
}
/**
* 拷贝依赖库到缓存目录
* @param osInfo
* @return
*/
private static List<File> getLibFiles(OsInfo osInfo,DeviceEnum deviceEnum){
try {
String device = getDevice(deviceEnum);
log.info("当前设备：{}", device);
//获取dll文件列表
List<DllItem> baseList = new ArrayList<>();
List<DllItem> jniList = new ArrayList<>();
InputStream propsInputStream = LoadNativeCore.class.getResourceAsStream(getPropertiesPath());
Props props = new Props();
props.load(propsInputStream);
String prefix = getPrefix();
props.forEach((keyObj, valuObj) -> {
String key = (String) keyObj;
String value = (String) valuObj;
DllItem dllItem = new DllItem();
dllItem.setKey(key);
if (key.contains("base")) {
if (value.contains("tennis")) {
dllItem.setValue(prefix + "base/" + device + "/" + value);
} else {
dllItem.setValue(prefix + "base/" + value);
}
baseList.add(dllItem);
} else {
dllItem.setValue(prefix + value);
jniList.add(dllItem);
}
});
//给dll文件排序
List<String> basePath = getSortedPath(baseList);
List<String> sdkPath = getSortedPath(jniList);
List<File> fileList = new ArrayList<>();
//拷贝文件到临时目录
for (String baseSo : basePath) {
fileList.add(extractLibrary(baseSo));
}
for (String sdkSo : sdkPath) {
fileList.add(extractLibrary(sdkSo));
}
return fileList;
} catch (Exception e) {
throw new FaceException("拷贝依赖库失败",e);
}
}
private static String getDevice(DeviceEnum deviceEnum) {
String device = "CPU";
if ("amd64".equals(getArch()) && deviceEnum != null) {
device = deviceEnum == DeviceEnum.GPU ? "GPU" : "CPU";
}
return device;
}
/**
* 返回路径文件前缀
*
* @return
*/
private static String getPrefix() {
String arch = getArch();
//aarch64
String os = SystemUtil.getOsInfo().getName();
//Windows操作系统
if (os != null && os.toLowerCase().startsWith("windows")) {
os = "/windows/";
} else if (os != null && os.toLowerCase().startsWith("linux")) {//Linux操作系统
os = "/linux/";
} else { //其它操作系统
//安卓 乌班图等等，先不写
return null;
}
// "/seetaface6/windows/amd64"
return "/" + SEETAFACE_LIB_DIR + os + arch + "/";
}
private static String getArch() {
String arch = SystemUtil.getOsInfo().getArch().toLowerCase();
if (arch.startsWith("amd64")
|| arch.startsWith("x86_64")
|| arch.startsWith("x86-64")
|| arch.startsWith("x64")) {
arch = "amd64";
} else if (arch.contains("aarch")) {
arch = "aarch64";
} else if (arch.contains("arm")) {
arch = "arm";
}
return arch;
}
/**
* 获取dll配置文件路径
*
* @return String
*/
private static String getPropertiesPath() {
return getPrefix() + PROPERTIES_FILE_NAME;
}
/**
* 拷贝依赖库到临时目录
* @param libPath
* @return
* @throws IOException
*/
private static File extractLibrary(String libPath) throws IOException {
String resourcePath = libPath;
try (InputStream in = NativeLoader.class.getResourceAsStream(resourcePath)) {
if (in == null) throw new FileNotFoundException(resourcePath);
Path path = Paths.get(resourcePath);
String fileName = path.getFileName().toString();
Path targetPath = seetaface6NativePath.resolve(fileName);
Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING);
log.info("copy target path success : {}", targetPath.toAbsolutePath().toString());
// 设置可执行权限
if (!SystemUtil.getOsInfo().getName().toLowerCase().contains("win")) {
targetPath.toFile().setExecutable(true);
}
return targetPath.toFile();
}
}
/**
* 将获得的配置进行排序 并生成路径
*
* @param list
* @return List<String>
*/
private static List<String> getSortedPath(List<DllItem> list) {
return list.stream().sorted(Comparator.comparing(dllItem -> {
int i = dllItem.getKey().lastIndexOf(".") + 1;
String substring = dllItem.getKey().substring(i);
return Integer.valueOf(substring);
})).map(DllItem::getValue).collect(Collectors.toList());
}
}

68
smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java Normal file
View File

@@ -0,0 +1,68 @@
package cn.smartjavaai.face.utils;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import cn.smartjavaai.common.utils.ImageUtils;
import com.seeta.sdk.SeetaImageData;
import com.seeta.sdk.SeetaPointF;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import java.awt.image.BufferedImage;
/**
* 人脸对齐
* @author Calvin
*/
public class FaceAlignUtils {
/**
* 根据目标点，进行旋转仿射变换
* Perform rotation and affine transformation based on the target 5 points
*
* @param src
* @param rot_mat
* @return
*/
public static Mat warpAffine(Mat src, Mat rot_mat) {
Mat rot = new Mat();
// 进行仿射变换，变换后大小为src的大小
// Perform affine transformation, the size after transformation is the same as the size of src
Scalar scalar = new Scalar(135, 133, 132);
Size size = new Size(512, 512);
Imgproc.warpAffine(src, rot, rot_mat, size, 0, 0, scalar);
return rot;
}
public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height) {
Mat rot = new Mat();
Size size = new Size(width, height);
Imgproc.warpAffine(src, rot, rot_mat, size);
return rot;
}
public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height, int flags) {
Mat rot = new Mat();
Size size = new Size(width, height);
Imgproc.warpAffine(src, rot, rot_mat, size, flags);
return rot;
}
public static SeetaImageData faceAlign(BufferedImage sourceImage, SeetaPointF[] pointFS) {
NDManager manager = NDManager.newBaseManager();
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(pointFS);
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine(OpenCVUtils.image2Mat(sourceImage), affine_matrix);
BufferedImage alignImage = OpenCVUtils.mat2Image(mat);
SeetaImageData imageData = new SeetaImageData(alignImage.getWidth(), alignImage.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(alignImage);
return imageData;
}
}

165
smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java
View File

@@ -3,12 +3,16 @@ package cn.smartjavaai.face.utils;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.BoundingBox;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.FaceModelConfig;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.SeetaImageData;
import com.seeta.sdk.SeetaPointF;
import com.seeta.sdk.SeetaRect;
import javax.imageio.ImageIO;
@@ -16,10 +20,8 @@ import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.*;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
@@ -49,11 +51,21 @@ public class FaceUtils {
while(iterator.hasNext()) {
DetectedObjects.DetectedObject result = (DetectedObjects.DetectedObject)iterator.next();
BoundingBox box = result.getBoundingBox();
int x = (int)(box.getBounds().getX() * (double)img.getWidth());
int y = (int)(box.getBounds().getY() * (double)img.getHeight());
int width = (int)(box.getBounds().getWidth() * (double)img.getWidth());
int height = (int)(box.getBounds().getHeight() * (double)img.getHeight());
List<Point> keyPoints = new ArrayList<Point>();
box.getBounds().getPath().forEach(point -> {
keyPoints.add(new Point(point.getX(), point.getY()));
});
int x = (int)(box.getBounds().getX() * img.getWidth());
int y = (int)(box.getBounds().getY() * img.getHeight());
int width = (int)(box.getBounds().getWidth() * img.getWidth());
int height = (int)(box.getBounds().getHeight() * img.getHeight());
// 修正边界，防止越界
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x + width > img.getWidth()) width = img.getWidth() - x;
if (y + height > img.getHeight()) height = img.getHeight() - y;
DetectionRectangle rectangle = new DetectionRectangle(x, y, width, height, detection.getProbabilities().get(index).floatValue());
rectangle.setKeyPoints(keyPoints);
rectangleList.add(rectangle);
index++;
}
@@ -66,18 +78,24 @@ public class FaceUtils {
* @param seetaResult
* @return
*/
public static DetectionResponse convertToDetectionResponse(SeetaRect[] seetaResult, FaceModelConfig config){
public static DetectionResponse convertToDetectionResponse(SeetaRect[] seetaResult, FaceModelConfig config,List<SeetaPointF[]> seetaPointFSList){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return null;
}
DetectionResponse detectionResponse = new DetectionResponse();
List<DetectionRectangle> rectangleList = new ArrayList<DetectionRectangle>();
for(SeetaRect rect : seetaResult){
for(int i = 0; i < seetaResult.length; i++){
SeetaRect rect = seetaResult[i];
SeetaPointF[] seetaPointFS = seetaPointFSList.get(i);
//过滤置信度
/*if(config.getConfidenceThreshold() > 0){
continue;
}*/
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height, 0);
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
rectangle.setKeyPoints(keyPoints);
rectangleList.add(rectangle);
}
detectionResponse.setRectangleList(rectangleList);
@@ -108,6 +126,10 @@ public class FaceUtils {
graphics.setColor(Color.RED);// 边框颜色
graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight());
drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4);
//绘制人脸关键点
if(rectangle.getKeyPoints() != null){
drawLandmarks(graphics, rectangle.getKeyPoints());
}
}
graphics.dispose();
ImageIO.write(sourceImage, "jpg", new File(savePath));
@@ -136,6 +158,10 @@ public class FaceUtils {
graphics.setColor(Color.RED);// 边框颜色
graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight());
drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4);
//绘制人脸关键点
if(rectangle.getKeyPoints() != null){
drawLandmarks(graphics, rectangle.getKeyPoints());
}
}
graphics.dispose();
return sourceImage;
@@ -163,5 +189,124 @@ public class FaceUtils {
g.drawString(text, x + padding, y + ascent);
}
/**
* 修正检测框
* @param rectangle
* @param imageWidth
* @param imageHeight
* @return
*/
public static DetectionRectangle correctRect(DetectionRectangle rectangle, int imageWidth, int imageHeight) {
int x = rectangle.getX();
int y = rectangle.getY();
int width = rectangle.getWidth();
int height = rectangle.getHeight();
// 修正x, y防止越界
if (x < 0) x = 0;
if (y < 0) y = 0;
// 宽高不能超出图片范围
if (x + width > imageWidth) {
width = imageWidth - x;
}
if (y + height > imageHeight) {
height = imageHeight - y;
}
// 防止最终 width 或 height 为负或为 0
if (width <= 0 || height <= 0) {
return null; // 无效区域
}
return new DetectionRectangle(x, y, width, height, rectangle.score);
}
/**
* 子图中人脸关键点坐标 - Coordinates of key points in the image
*
* @param points
* @return
*/
public static double[][] facePoints(List<Point> points) {
// 图中关键点坐标 - Coordinates of key points in the image
// 1. left_eye_x , left_eye_y
// 2. right_eye_x , right_eye_y
// 3. nose_x , nose_y
// 4. left_mouth_x , left_mouth_y
// 5. right_mouth_x , right_mouth_y
double[][] pointsArray = new double[5][2]; // 保存人脸关键点 - Save facial key points
int i = 0;
for (Point point : points) {
pointsArray[i][0] = point.getX();
pointsArray[i][1] = point.getY();
i++;
}
return pointsArray;
}
/**
* 子图中人脸关键点坐标 - Coordinates of key points in the image
*
* @param pointFS
* @return
*/
public static double[][] facePoints(SeetaPointF[] pointFS) {
// 图中关键点坐标 - Coordinates of key points in the image
// 1. left_eye_x , left_eye_y
// 2. right_eye_x , right_eye_y
// 3. nose_x , nose_y
// 4. left_mouth_x , left_mouth_y
// 5. right_mouth_x , right_mouth_y
double[][] pointsArray = new double[5][2]; // 保存人脸关键点 - Save facial key points
int i = 0;
for (SeetaPointF point : pointFS) {
pointsArray[i][0] = point.getX();
pointsArray[i][1] = point.getY();
i++;
}
return pointsArray;
}
/**
* 512x512的目标点 - Target point of 512x512
* standard 5 landmarks for FFHQ faces with 512 x 512
*
* @param manager
* @return
*/
public static NDArray faceTemplate512x512(NDManager manager) {
double[][] coord5point = {
{192.98138, 239.94708}, // 512x512的目标点 - Target point of 512x512
{318.90277, 240.1936},
{256.63416, 314.01935},
{201.26117, 371.41043},
{313.08905, 371.15118}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* bgr转图片
* @return 图片
*/
public static BufferedImage toBufferedImage(SeetaImageData seetaImageData) {
int type = BufferedImage.TYPE_3BYTE_BGR;
BufferedImage image = new BufferedImage(seetaImageData.width, seetaImageData.height, type);
image.getRaster().setDataElements(0, 0, seetaImageData.width, seetaImageData.height, seetaImageData.data);
return image;
}
/**
* 绘制人脸关键点
* @param g
* @param keyPoints
*/
private static void drawLandmarks(Graphics2D g, List<Point> keyPoints) {
g.setColor(new Color(246, 96, 0));
BasicStroke bStroke = new BasicStroke(4.0F, 0, 0);
g.setStroke(bStroke);
for (Point point : keyPoints){
g.drawRect((int)point.getX(), (int)point.getY(), 2, 2);
}
}
}

119
smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/OpenCVUtils.java Normal file
View File

@@ -0,0 +1,119 @@
package cn.smartjavaai.face.utils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.imgproc.Imgproc;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
/**
* OpenCV 工具类
*/
public class OpenCVUtils {
/**
* canny算法，边缘检测
*
* @param src
* @return
*/
public static Mat canny(Mat src) {
Mat mat = src.clone();
Imgproc.Canny(src, mat, 100, 200);
return mat;
}
/**
* 画线
*
* @param mat
* @param point1
* @param point2
*/
public static void line(Mat mat, Point point1, Point point2) {
Imgproc.line(mat, point1, point2, new Scalar(255, 255, 255), 1);
}
/**
* NDArray to opencv_core.Mat
*
* @param manager
* @param srcPoints
* @param dstPoints
* @return
*/
public static Mat toOpenCVMat(NDManager manager, NDArray srcPoints, NDArray dstPoints) {
NDArray svdMat = SVDUtils.transformationFromPoints(manager, srcPoints, dstPoints);
double[] doubleArray = svdMat.toDoubleArray();
Mat newSvdMat = new Mat(2, 3, CvType.CV_64F);
for (int i = 0; i < 2; i++) {
for (int j = 0; j < 3; j++) {
newSvdMat.put(i, j, doubleArray[i * 3 + j]);
}
}
return newSvdMat;
}
/**
* double[][] points array to Mat
* @param points
* @return
*/
public static Mat toOpenCVMat(double[][] points) {
Mat mat = new Mat(5, 2, CvType.CV_64F);
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
mat.put(i, j, points[i * 5 + j]);
}
}
return mat;
}
/**
* 变换矩阵的逆矩阵
*
* @param src
* @return
*/
public static Mat invertAffineTransform(Mat src) {
Mat dst = src.clone();
Imgproc.invertAffineTransform(src, dst);
return dst;
}
/**
* Mat to BufferedImage
*
* @param mat
* @return
*/
public static BufferedImage mat2Image(Mat mat) {
int width = mat.width();
int height = mat.height();
byte[] data = new byte[width * height * (int) mat.elemSize()];
Imgproc.cvtColor(mat, mat, 4);
mat.get(0, 0, data);
BufferedImage ret = new BufferedImage(width, height, 5);
ret.getRaster().setDataElements(0, 0, width, height, data);
return ret;
}
/**
* BufferedImage to Mat
*
* @param img
* @return
*/
public static Mat image2Mat(BufferedImage img) {
int width = img.getWidth();
int height = img.getHeight();
byte[] data = ((DataBufferByte) img.getRaster().getDataBuffer()).getData();
Mat mat = new Mat(height, width, CvType.CV_8UC3);
mat.put(0, 0, data);
return mat;
}
}

119
smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/SVDUtils.java Normal file
View File

@@ -0,0 +1,119 @@
package cn.smartjavaai.face.utils;
import Jama.Matrix;
import Jama.SingularValueDecomposition;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
/**
* 仿射变换处理工具
*/
public class SVDUtils {
/**
* 计算仿射变换矩阵
* Calculate affine transformation matrix
*
* @param manager
* @param points1
* @param points2
* @return
*/
public static NDArray transformationFromPoints(
NDManager manager, NDArray points1, NDArray points2) {
// 按列计算均值
// Calculate column-wise mean
NDArray c1 = points1.mean(new int[]{0}); // axis=0 列操作 - axis=0 column operation
NDArray c2 = points2.mean(new int[]{0}); // axis=0 列操作 - axis=0 column operation
// 按列减去均值
// Subtract column-wise mean
points1 = points1.sub(c1);
points2 = points2.sub(c2);
// 计算全局标准差
// Calculate global standard deviation
double s1 = std(points1);
double s2 = std(points2);
// 矩阵除以全局标准差
// Matrix divided by global standard deviation
NDArray djl_s1 = manager.create(s1);
NDArray djl_s2 = manager.create(s2);
points1 = points1.div(djl_s1);
points2 = points2.div(djl_s2);
double[] points1D = points1.toDoubleArray();
double[] points2D = points2.toDoubleArray();
// DJL 格式转换成Jamma格式
// Convert DJL format to Jama format
double[][] m1 = new double[5][2];
double[][] m2 = new double[5][2];
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
m1[i][j] = points1D[i * 2 + j];
}
}
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
m2[i][j] = points2D[i * 2 + j];
}
}
Matrix p1 = new Matrix(m1);
Matrix p2 = new Matrix(m2);
// 进行奇异值分解
// Perform singular value decomposition
Matrix p3 = p1.transpose().times(p2);
SingularValueDecomposition s = p3.svd();
Matrix U = s.getU();
Matrix S = s.getS();
Matrix V = s.getV();
// TODO 为什么第2列的符号是反的？
// Why is the sign of the second column opposite?
m1 = U.getArray();
m1[0][1] = -m1[0][1];
m1[1][1] = -m1[1][1];
m2 = V.getArray();
m2[0][1] = -m2[0][1];
m2[1][1] = -m2[1][1];
Matrix R = (U.times(V)).transpose();
double[][] rArray = R.getArray();
NDArray newR = manager.create(rArray);
// np.vstack([np.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)), np.matrix([0.,0., 1.])])
// (s2 / s1) * R
NDArray leftPart = djl_s2.div(djl_s1).mul(newR);
// c2.T - (s2 / s1) * R * c1.T)
NDArray rightPart = c2.reshape(2, 1).sub(leftPart.matMul(c1.reshape(2, 1)));
// numpy.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T))
NDArray upPart = leftPart.concat(rightPart, 1);
// np.matrix([0.,0., 1.])
double[] downArray = {0d, 0d, 1d};
NDArray downPart = manager.create(downArray).reshape(1, 3);
NDArray all = upPart.concat(downPart, 0);
// System.out.println("all: " + all);
return upPart;
}
/**
* 计算全局标准差
* Calculate global standard deviation
*
* @param points
* @return
*/
public static double std(NDArray points) {
points = points.square();
double[] doubleResult = points.toDoubleArray();
double std = 0;
for (int i = 0; i < doubleResult.length; i++) {
std = std + doubleResult[i];
}
std = (float) Math.sqrt(std / doubleResult.length);
return std;
}
}