commit 28f55e85ccccc730836d1b280970c825f4a6bcf5 Author: dengwenjie <775747758@qq.com> Date: Fri Feb 21 11:26:58 2025 +0800 初始提交 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..45c3a01 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.idea +.idea/ +target +log +*.iml +/.settings/ +/logging.file_IS_UNDEFINED/ + diff --git a/output.jpg b/output.jpg new file mode 100644 index 0000000..974fd32 Binary files /dev/null and b/output.jpg differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..a4606bf --- /dev/null +++ b/pom.xml @@ -0,0 +1,110 @@ + + + 4.0.0 + + ink.numberone + smartjavaai-parent + 1.0.0 + pom + + smartjavaai-face + smartjavaai-common + + + + 11 + 11 + + UTF-8 + 0.32.0-SNAPSHOT + + + + + + ai.djl + bom + ${djl.version} + pom + import + + + + + + commons-cli + commons-cli + 1.9.0 + + + commons-io + commons-io + 2.17.0 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.1 + + + ai.djl + api + + + ai.djl + basicdataset + + + ai.djl + model-zoo + + + ai.djl.timeseries + timeseries + + + ai.djl.huggingface + tokenizers + + + ai.djl.audio + audio + + + + ai.djl.mxnet + mxnet-model-zoo + + + + ai.djl.pytorch + pytorch-model-zoo + + + + ai.djl.tensorflow + tensorflow-model-zoo + + + + ai.djl.onnxruntime + onnxruntime-engine + + + org.testng + testng + 7.10.2 + test + + + + commons-beanutils + commons-beanutils + 1.9.4 + + + + diff --git a/smartjavaai-common/pom.xml b/smartjavaai-common/pom.xml new file mode 100644 index 0000000..7abd336 --- /dev/null +++ b/smartjavaai-common/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + ink.numberone + smartjavaai-parent + 1.0.0 + + + smartjavaai-common + + + 8 + 8 + UTF-8 + + + diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Point.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Point.java new file mode 100644 index 0000000..0dbe960 --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Point.java @@ -0,0 +1,42 @@ +package cn.smartjavaai.common.entity; + +import ai.djl.util.JsonUtils; + +import java.io.Serializable; + +/** + * 点 + * @author dwj + * @date 2025/2/19 + */ +public class Point implements Serializable { + private static final long serialVersionUID = 1L; + private int x; + private int y; + + public Point(int x, int y) { + this.x = x; + this.y = y; + } + + public int getX() { + return x; + } + + public void setX(int x) { + this.x = x; + } + + public int getY() { + return y; + } + + public void setY(int y) { + this.y = y; + } + + @Override + public String toString() { + return JsonUtils.GSON_COMPACT.toJson(this); + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Rectangle.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Rectangle.java new file mode 100644 index 0000000..f5a8655 --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Rectangle.java @@ -0,0 +1,50 @@ +package cn.smartjavaai.common.entity; + +import java.util.List; + +/** + * 矩形区域 + * @author dwj + * @date 2025/2/19 + */ +public class Rectangle { + + /** + * 矩形区域点集合 + */ + List pointList; + + /** + * 矩形区域宽度 + */ + int width; + + /** + * 矩形区域高度 + */ + int height; + + public int getWidth() { + return width; + } + + public void setWidth(int width) { + this.width = width; + } + + public int getHeight() { + return height; + } + + public void setHeight(int height) { + this.height = height; + } + + public List getPointList() { + return pointList; + } + + public void setPointList(List pointList) { + this.pointList = pointList; + } +} diff --git a/smartjavaai-face/pom.xml b/smartjavaai-face/pom.xml new file mode 100644 index 0000000..5abae2d --- /dev/null +++ b/smartjavaai-face/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + ink.numberone + smartjavaai-parent + 1.0.0 + + + smartjavaai-face + + + 8 + 8 + UTF-8 + + + + + + ink.numberone + smartjavaai-common + ${project.version} + + + + + + + + diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/AbstractFaceAlgorithm.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/AbstractFaceAlgorithm.java new file mode 100644 index 0000000..366cfe4 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/AbstractFaceAlgorithm.java @@ -0,0 +1,50 @@ +package cn.smartjavaai.face; + +import java.io.InputStream; + +/** + * 人脸识别算法 + * @author dwj + * @date 2025/2/20 + */ +public abstract class AbstractFaceAlgorithm implements FaceAlgorithm{ + @Override + public void loadModel(ModelConfig config) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public FaceDetectedResult detect(String imagePath) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public FaceDetectedResult detect(InputStream imageInputStream) throws Exception { + return null; + } + + @Override + public float[] featureExtraction(String imagePath) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public float[] featureExtraction(InputStream inputStream) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public float calculSimilar(float[] feature1, float[] feature2) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public float featureComparison(String imagePath1, String imagePath2) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + @Override + public float featureComparison(InputStream inputStream1, InputStream inputStream2) throws Exception { + throw new UnsupportedOperationException("默认不支持该功能"); + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithm.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithm.java new file mode 100644 index 0000000..7bff051 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithm.java @@ -0,0 +1,83 @@ +package cn.smartjavaai.face; + +import ai.djl.MalformedModelException; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.translate.TranslateException; + +import java.io.IOException; +import java.io.InputStream; + +/** + * 人脸识别算法 + * @author dwj + * @date 2025/2/19 + */ +public interface FaceAlgorithm { + + /** + * 加载模型 + * @param config + * @throws Exception + */ + void loadModel(ModelConfig config) throws Exception; // 加载模型 + + /** + * 人脸检测 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + FaceDetectedResult detect(String imagePath) throws Exception; + + /** + * 人脸检测 + * @param imageInputStream 图片输入流 + * @return + * @throws Exception + */ + FaceDetectedResult detect(InputStream imageInputStream) throws Exception; + + /** + * 特征提取 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + float[] featureExtraction(String imagePath) throws Exception; + + /** + * 特征提取 + * @param inputStream 输入流 + * @return + * @throws Exception + */ + float[] featureExtraction(InputStream inputStream) throws Exception; + + /** + * 计算相似度 + * @param feature1 图1特征 + * @param feature2 图2特征 + * @return + * @throws Exception + */ + float calculSimilar(float[] feature1, float[] feature2) throws Exception; + + /** + * 特征比较 + * @param imagePath1 图1路径 + * @param imagePath2 图2路径 + * @return + * @throws Exception + */ + float featureComparison(String imagePath1, String imagePath2) throws Exception; + + /** + * 特征比较 + * @param inputStream1 图1输入流 + * @param inputStream2 图2输入流 + * @return + * @throws Exception + */ + float featureComparison(InputStream inputStream1, InputStream inputStream2) throws Exception; + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithmFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithmFactory.java new file mode 100644 index 0000000..0051830 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceAlgorithmFactory.java @@ -0,0 +1,95 @@ +package cn.smartjavaai.face; + +import cn.smartjavaai.face.algo.RetinaFace; +import cn.smartjavaai.face.algo.UltraLightFastGenericFace; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 人脸算法工厂 + * @author dwj + * @date 2025/2/19 + */ +public class FaceAlgorithmFactory { + + /** + * 算法注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + /** + * 注册算法 + * @param name + * @param clazz + */ + public static void registerAlgorithm(String name, Class clazz) { + registry.put(name.toLowerCase(), clazz); + } + + /** + * 使用ModelConfig创建算法 + * @param config + * @return + * @throws Exception + */ + public static FaceAlgorithm createFaceAlgorithm(ModelConfig config) throws Exception { + Class clazz = registry.get(config.getAlgorithmName().toLowerCase()); + if(clazz == null){ + throw new IllegalArgumentException("Unsupported algorithm"); + } + FaceAlgorithm algorithm = (FaceAlgorithm) clazz.newInstance(); + algorithm.loadModel(config); + return algorithm; + } + + /** + * 创建默认算法 + * @return + * @throws Exception + */ + public static FaceAlgorithm createFaceAlgorithm() throws Exception { + // 初始化配置 + ModelConfig config = new ModelConfig(); + config.setAlgorithmName("retinaface"); + config.setConfidenceThreshold(FaceConfig.DEFAULT_CONFIDENCE_THRESHOLD); + config.setMaxFaceCount(FaceConfig.MAX_FACE_LIMIT); + config.setNmsThresh(FaceConfig.NMS_THRESHOLD); + Class clazz = registry.get(config.getAlgorithmName().toLowerCase()); + if(clazz == null){ + throw new IllegalArgumentException("Unsupported algorithm"); + } + FaceAlgorithm algorithm = (FaceAlgorithm) clazz.newInstance(); + algorithm.loadModel(config); + return algorithm; + } + + /** + * 创建轻量级算法 + * @return + * @throws Exception + */ + public static FaceAlgorithm createLightFaceAlgorithm() throws Exception { + // 初始化配置 + ModelConfig config = new ModelConfig(); + config.setAlgorithmName("ultralightfastgenericface"); + config.setConfidenceThreshold(FaceConfig.DEFAULT_CONFIDENCE_THRESHOLD); + config.setMaxFaceCount(FaceConfig.MAX_FACE_LIMIT); + config.setNmsThresh(FaceConfig.NMS_THRESHOLD); + Class clazz = registry.get(config.getAlgorithmName().toLowerCase()); + if(clazz == null){ + throw new IllegalArgumentException("Unsupported algorithm"); + } + FaceAlgorithm algorithm = (FaceAlgorithm) clazz.newInstance(); + algorithm.loadModel(config); + return algorithm; + } + + // 初始化默认算法 + static { + registerAlgorithm("retinaface", RetinaFace.class); + registerAlgorithm("ultralightfastgenericface", UltraLightFastGenericFace.class); + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceConfig.java new file mode 100644 index 0000000..eabcc6c --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceConfig.java @@ -0,0 +1,26 @@ +package cn.smartjavaai.face; + +/** + * @author dwj + * @date 2025/2/20 + */ +public class FaceConfig { + + /** + * 置信度阈值 + */ + public static final float DEFAULT_CONFIDENCE_THRESHOLD = 0.85F; + + /** + * 每张特征图保留的最大候选框数量 + */ + public static final int MAX_FACE_LIMIT = 5000; + + /** + * nms阈值:控制重叠框的合并程度 + */ + public static final float NMS_THRESHOLD = 0.45F; + + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectedResult.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectedResult.java new file mode 100644 index 0000000..24f80ad --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectedResult.java @@ -0,0 +1,41 @@ +package cn.smartjavaai.face; + + +import cn.smartjavaai.common.entity.Rectangle; + +import java.util.List; + +/** + * 人脸检测结果 + * @author dwj + * @date 2025/2/19 + */ +public class FaceDetectedResult { + + + /** + * 置信度 + */ + private List probabilities; + + /** + * 人脸框 + */ + private List rectangles; + + public List getProbabilities() { + return probabilities; + } + + public void setProbabilities(List probabilities) { + this.probabilities = probabilities; + } + + public List getRectangles() { + return rectangles; + } + + public void setRectangles(List rectangles) { + this.rectangles = rectangles; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectionTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectionTranslator.java new file mode 100644 index 0000000..f70b567 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/FaceDetectionTranslator.java @@ -0,0 +1,212 @@ +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package cn.smartjavaai.face; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.*; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDArrays; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Face Detection Translator + */ +public class FaceDetectionTranslator implements Translator { + + private double confThresh; + private double nmsThresh; + private int topK; + private double[] variance; + private int[][] scales; + private int[] steps; + + public FaceDetectionTranslator( + double confThresh, + double nmsThresh, + double[] variance, + int topK, + int[][] scales, + int[] steps) { + this.confThresh = confThresh; + this.nmsThresh = nmsThresh; + this.variance = variance; + this.topK = topK; + this.scales = scales; + this.steps = steps; + } + + /** {@inheritDoc} */ + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + + ctx.setAttachment("width", input.getWidth()); + ctx.setAttachment("height", input.getHeight()); + + NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW RGB -> BGR + // The network by default takes float32 + if (!array.getDataType().equals(DataType.FLOAT32)) { + array = array.toType(DataType.FLOAT32, false); + } + NDArray mean = + ctx.getNDManager().create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean); + return new NDList(array); + } + + /** {@inheritDoc} */ + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + + int width = (int) ctx.getAttachment("width"); + int height = (int) ctx.getAttachment("height"); + + NDManager manager = ctx.getNDManager(); + double scaleXY = variance[0]; + double scaleWH = variance[1]; + + NDArray prob = list.get(1).get(":, 1:"); + prob = + NDArrays.stack( + new NDList( + prob.argMax(1).toType(DataType.FLOAT32, false), + prob.max(new int[] {1}))); + + NDArray boxRecover = boxRecover(manager, width, height, scales, steps); + NDArray boundingBoxes = list.get(0); + NDArray bbWH = boundingBoxes.get(":, 2:").mul(scaleWH).exp().mul(boxRecover.get(":, 2:")); + NDArray bbXY = + boundingBoxes + .get(":, :2") + .mul(scaleXY) + .mul(boxRecover.get(":, 2:")) + .add(boxRecover.get(":, :2")) + .sub(bbWH.mul(0.5f)); + + boundingBoxes = NDArrays.concat(new NDList(bbXY, bbWH), 1); + + NDArray landms = list.get(2); + landms = decodeLandm(landms, boxRecover, scaleXY); + + // filter the result below the threshold + NDArray cutOff = prob.get(1).gt(confThresh); + boundingBoxes = boundingBoxes.transpose().booleanMask(cutOff, 1).transpose(); + landms = landms.transpose().booleanMask(cutOff, 1).transpose(); + prob = prob.booleanMask(cutOff, 1); + + // start categorical filtering + long[] order = prob.get(1).argSort().get(":" + topK).toLongArray(); + prob = prob.transpose(); + List retNames = new ArrayList<>(); + List retProbs = new ArrayList<>(); + List retBB = new ArrayList<>(); + + Map> recorder = new ConcurrentHashMap<>(); + + for (int i = order.length - 1; i >= 0; i--) { + long currMaxLoc = order[i]; + float[] classProb = prob.get(currMaxLoc).toFloatArray(); + int classId = (int) classProb[0]; + double probability = classProb[1]; + + double[] boxArr = boundingBoxes.get(currMaxLoc).toDoubleArray(); + double[] landmsArr = landms.get(currMaxLoc).toDoubleArray(); + Rectangle rect = new Rectangle(boxArr[0], boxArr[1], boxArr[2], boxArr[3]); + List boxes = recorder.getOrDefault(classId, new ArrayList<>()); + boolean belowIoU = true; + for (BoundingBox box : boxes) { + if (box.getIoU(rect) > nmsThresh) { + belowIoU = false; + break; + } + } + if (belowIoU) { + List keyPoints = new ArrayList<>(); + for (int j = 0; j < 5; j++) { // 5 face landmarks + double x = landmsArr[j * 2]; + double y = landmsArr[j * 2 + 1]; + keyPoints.add(new Point(x * width, y * height)); + } + Landmark landmark = + new Landmark(boxArr[0], boxArr[1], boxArr[2], boxArr[3], keyPoints); + + boxes.add(landmark); + recorder.put(classId, boxes); + String className = "Face"; // classes.get(classId) + retNames.add(className); + retProbs.add(probability); + retBB.add(landmark); + } + } + + return new DetectedObjects(retNames, retProbs, retBB); + } + + private NDArray boxRecover( + NDManager manager, int width, int height, int[][] scales, int[] steps) { + int[][] aspectRatio = new int[steps.length][2]; + for (int i = 0; i < steps.length; i++) { + int wRatio = (int) Math.ceil((float) width / steps[i]); + int hRatio = (int) Math.ceil((float) height / steps[i]); + aspectRatio[i] = new int[] {hRatio, wRatio}; + } + + List defaultBoxes = new ArrayList<>(); + + for (int idx = 0; idx < steps.length; idx++) { + int[] scale = scales[idx]; + for (int h = 0; h < aspectRatio[idx][0]; h++) { + for (int w = 0; w < aspectRatio[idx][1]; w++) { + for (int i : scale) { + double skx = i * 1.0 / width; + double sky = i * 1.0 / height; + double cx = (w + 0.5) * steps[idx] / width; + double cy = (h + 0.5) * steps[idx] / height; + defaultBoxes.add(new double[] {cx, cy, skx, sky}); + } + } + } + } + + double[][] boxes = new double[defaultBoxes.size()][defaultBoxes.get(0).length]; + for (int i = 0; i < defaultBoxes.size(); i++) { + boxes[i] = defaultBoxes.get(i); + } + return manager.create(boxes).clip(0.0, 1.0); + } + + // decode face landmarks, 5 points per face + private NDArray decodeLandm(NDArray pre, NDArray priors, double scaleXY) { + NDArray point1 = + pre.get(":, :2").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2")); + NDArray point2 = + pre.get(":, 2:4").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2")); + NDArray point3 = + pre.get(":, 4:6").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2")); + NDArray point4 = + pre.get(":, 6:8").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2")); + NDArray point5 = + pre.get(":, 8:10").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2")); + return NDArrays.concat(new NDList(point1, point2, point3, point4, point5), 1); + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/ModelConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/ModelConfig.java new file mode 100644 index 0000000..c378987 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/ModelConfig.java @@ -0,0 +1,62 @@ +package cn.smartjavaai.face; + +/** + * 模型配置 + * @author dwj + * @date 2025/2/19 + */ + +public class ModelConfig { + + /** + * 人脸算法名称 + */ + private String algorithmName; + + /** + * 置信度阈值 + */ + private double confidenceThreshold; + + /** + * 非极大抑制阈值 作用：消除重叠检测框，保留最优结果 + */ + private double nmsThresh; + + /** + * 最大检测人脸数量 + */ + private int maxFaceCount; + + public String getAlgorithmName() { + return algorithmName; + } + + public void setAlgorithmName(String algorithmName) { + this.algorithmName = algorithmName; + } + + public double getConfidenceThreshold() { + return confidenceThreshold; + } + + public void setConfidenceThreshold(double confidenceThreshold) { + this.confidenceThreshold = confidenceThreshold; + } + + public double getNmsThresh() { + return nmsThresh; + } + + public void setNmsThresh(double nmsThresh) { + this.nmsThresh = nmsThresh; + } + + public int getMaxFaceCount() { + return maxFaceCount; + } + + public void setMaxFaceCount(int maxFaceCount) { + this.maxFaceCount = maxFaceCount; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/Test.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/Test.java new file mode 100644 index 0000000..286cfcb --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/Test.java @@ -0,0 +1,62 @@ +package cn.smartjavaai.face; + +import cn.smartjavaai.common.entity.Rectangle; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileInputStream; + +/** + * @author dwj + * @date 2025/2/19 + */ +public class Test { + + public static void main(String[] args) throws Exception { + // 初始化配置 + ModelConfig config = new ModelConfig(); + config.setAlgorithmName("retinaface"); + //config.setAlgorithmName("ultralightfastgenericface"); + //config.setModelPath("/Users/wenjie/Documents/idea_workplace/SmartJavaAI/models/retinaface.pt"); + config.setConfidenceThreshold(0.85f); + config.setMaxFaceCount(100); + config.setNmsThresh(0.45f); + + FaceAlgorithm currentAlgorithm = FaceAlgorithmFactory.createFaceAlgorithm(config); + File imageFile = new File("/Users/wenjie/Downloads/djl-master/examples/src/test/resources/largest_selfie.jpg"); + FaceDetectedResult result = currentAlgorithm.detect("/Users/wenjie/Downloads/djl-master/examples/src/test/resources/largest_selfie.jpg"); + //FaceDetectedResult result = currentAlgorithm.detect(new FileInputStream(imageFile)); + + + // 1. 加载原始图片 + File input = new File("/Users/wenjie/Downloads/djl-master/examples/src/test/resources/largest_selfie.jpg"); + BufferedImage image = ImageIO.read(input); + + // 2. 创建绘图上下文 + Graphics2D graphics = image.createGraphics(); + + // 3. 配置绘制参数 + graphics.setColor(Color.RED); // 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + + for(Rectangle rectangle : result.getRectangles()){ + // 4. 绘制矩形（左上角坐标x=50,y=100，宽200，高150） + graphics.drawRect(rectangle.getPointList().get(0).getX(), + rectangle.getPointList().get(0).getY(), rectangle.getWidth(), rectangle.getHeight()); + } + + + + // 5. 释放资源并保存 + graphics.dispose(); + ImageIO.write(image, "jpg", new File("output.jpg")); + + + System.out.println("111111"); + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/RetinaFace.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/RetinaFace.java new file mode 100644 index 0000000..951214f --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/RetinaFace.java @@ -0,0 +1,278 @@ +package cn.smartjavaai.face.algo; + +import ai.djl.MalformedModelException; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; +import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.Rectangle; +import cn.smartjavaai.face.*; +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.compress.utils.Lists; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * RetinaFace实现 + * @author dwj + * @date 2025/2/19 + */ +public class RetinaFace extends AbstractFaceAlgorithm { + + private Criteria criteria; + + /** + * 特征图层的基础缩放比例 + */ + public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}}; + /** + * 特征图相对于原图的采样步长 + */ + public static final int[] steps = {8, 16, 32}; + /** + * 缩放系数 + */ + public static final double[] variance = {0.1f, 0.2f}; + + /** + * 加载模型 + * @param config + */ + @Override + public void loadModel(ModelConfig config) { + FaceDetectionTranslator translator = + new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, config.getMaxFaceCount(), scales, steps); + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls("https://resources.djl.ai/test-models/pytorch/retinaface.zip") + //.optModelPath(modelPath) + // Load model from local file, e.g: + .optModelName("retinaface") // specify model file prefix + .optTranslator(translator) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + } + + /** + * 检测人脸 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + @Override + public FaceDetectedResult detect(String imagePath) throws Exception{ + Path facePath = Paths.get(imagePath); + Image img = ImageFactory.getInstance().fromFile(facePath); + try (ZooModel model = criteria.loadModel(); + Predictor predictor = model.newPredictor()) { + DetectedObjects detection = predictor.predict(img); + return convertToFaceDetectedResult(detection,img); + } + } + + /** + * 检测人脸 + * @param imageInputStream 图片流 + * @return + * @throws Exception + */ + @Override + public FaceDetectedResult detect(InputStream imageInputStream) throws Exception { + Image img = ImageFactory.getInstance().fromInputStream(imageInputStream); + try (ZooModel model = criteria.loadModel(); + Predictor predictor = model.newPredictor()) { + DetectedObjects detection = predictor.predict(img); + return convertToFaceDetectedResult(detection,img); + /*saveBoundingBoxImage(img, detection); + return detection;*/ + } + } + + /** + * 转换为FaceDetectedResult + * @param detection + * @param img + * @return + */ + private FaceDetectedResult convertToFaceDetectedResult(DetectedObjects detection, Image img){ + FaceDetectedResult faceDetectedResult = new FaceDetectedResult(); + List probabilities = new ArrayList<>(detection.getProbabilities()); + List detectedObjectList = detection.items(); + List RectangleList = detectedObjectList.parallelStream() + .map(obj -> { + Rectangle rectangle = new Rectangle(); + List pointList = new ArrayList<>(); + ai.djl.modality.cv.output.Rectangle rectangleDjl = obj.getBoundingBox().getBounds(); + int x = (int)(rectangleDjl.getX() * (double)img.getWidth()); + int y = (int)(rectangleDjl.getY() * (double)img.getHeight()); + int width = (int)(rectangleDjl.getWidth() * (double)img.getWidth()); + int height = (int)(rectangleDjl.getHeight() * (double)img.getHeight()); + pointList.add(new Point(x,y)); + pointList.add(new Point(x + width,y)); + pointList.add(new Point(x,y + height)); + pointList.add(new Point(x + width,y + height)); + rectangle.setPointList(pointList); + rectangle.setHeight(height); + rectangle.setWidth(width); + return rectangle; + }) + .collect(Collectors.toList()); + faceDetectedResult.setProbabilities(probabilities); + faceDetectedResult.setRectangles(RectangleList); + return faceDetectedResult; + } + + /** + * 特征提取 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + @Override + public float[] featureExtraction(String imagePath) throws Exception { + Path imageFile = Paths.get(imagePath); + Image img = ImageFactory.getInstance().fromFile(imageFile); + img.getWrappedImage(); + List mean = + Arrays.asList( + 127.5f / 255.0f, + 127.5f / 255.0f, + 127.5f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelUrls( + "https://resources.djl.ai/test-models/pytorch/face_feature.zip") + .optModelName("face_feature") // specify model file prefix + .optArgument("normalize", normalize) + .optTranslatorFactory(new ImageFeatureExtractorFactory()) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + + try (ZooModel model = criteria.loadModel()) { + Predictor predictor = model.newPredictor(); + return predictor.predict(img); + } + } + + /** + * 特征提取 + * @param inputStream 输入流 + * @return + * @throws Exception + */ + @Override + public float[] featureExtraction(InputStream inputStream) throws Exception { + Image img = ImageFactory.getInstance().fromInputStream(inputStream); + img.getWrappedImage(); + List mean = + Arrays.asList( + 127.5f / 255.0f, + 127.5f / 255.0f, + 127.5f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelUrls( + "https://resources.djl.ai/test-models/pytorch/face_feature.zip") + .optModelName("face_feature") // specify model file prefix + .optArgument("normalize", normalize) + .optTranslatorFactory(new ImageFeatureExtractorFactory()) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + + try (ZooModel model = criteria.loadModel()) { + Predictor predictor = model.newPredictor(); + return predictor.predict(img); + } + } + + /** + * 计算相似度 + * @param feature1 图1特征 + * @param feature2 图2特征 + * @return + * @throws Exception + */ + @Override + public float calculSimilar(float[] feature1, float[] feature2) throws Exception { + float ret = 0.0f; + float mod1 = 0.0f; + float mod2 = 0.0f; + int length = feature1.length; + for (int i = 0; i < length; ++i) { + ret += feature1[i] * feature2[i]; + mod1 += feature1[i] * feature1[i]; + mod2 += feature2[i] * feature2[i]; + } + return (float) ((ret / Math.sqrt(mod1) / Math.sqrt(mod2) + 1) / 2.0f); + } + + /** + * 特征比较 + * @param imagePath1 图1路径 + * @param imagePath2 图2路径 + * @return + * @throws Exception + */ + @Override + public float featureComparison(String imagePath1, String imagePath2) throws Exception { + float[] feature1 = featureExtraction(imagePath1); + float[] feature2 = featureExtraction(imagePath2); + return calculSimilar(feature1, feature2); + } + + /** + * 特征比较 + * @param inputStream1 图1输入流 + * @param inputStream2 图2输入流 + * @return + * @throws Exception + */ + @Override + public float featureComparison(InputStream inputStream1, InputStream inputStream2) throws Exception { + float[] feature1 = featureExtraction(inputStream1); + float[] feature2 = featureExtraction(inputStream2); + return calculSimilar(feature1, feature2); + } + + /*@Override + public float[] recognize(FaceRegion region) { + return new float[0]; + }*/ + + /*@Override + public void loadModel(ModelConfig config) throws Exception { + + }*/ +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/UltraLightFastGenericFace.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/UltraLightFastGenericFace.java new file mode 100644 index 0000000..169c291 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/algo/UltraLightFastGenericFace.java @@ -0,0 +1,270 @@ +package cn.smartjavaai.face.algo; + +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.Rectangle; +import cn.smartjavaai.face.*; + +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author dwj + * @date 2025/2/19 + */ +public class UltraLightFastGenericFace extends AbstractFaceAlgorithm { + + + private Criteria criteria; + + /** + * 特征图层的基础缩放比例 + */ + private static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}}; + /** + * 特征图相对于原图的采样步长 + */ + private static final int[] steps = {8, 16, 32, 64}; + /** + * 缩放系数 + */ + private static final double[] variance = {0.1f, 0.2f}; + + + + + /** + * 加载模型 + * @param config + */ + @Override + public void loadModel(ModelConfig config) { + FaceDetectionTranslator translator = + new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, config.getMaxFaceCount(), scales, steps); + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls("https://resources.djl.ai/test-models/pytorch/ultranet.zip") + .optTranslator(translator) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + } + + /** + * 检测人脸 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + @Override + public FaceDetectedResult detect(String imagePath) throws Exception{ + Path facePath = Paths.get(imagePath); + Image img = ImageFactory.getInstance().fromFile(facePath); + try (ZooModel model = criteria.loadModel(); + Predictor predictor = model.newPredictor()) { + DetectedObjects detection = predictor.predict(img); + return convertToFaceDetectedResult(detection,img); + } + } + + /** + * 检测人脸 + * @param imageInputStream 图片输入流 + * @return + * @throws Exception + */ + @Override + public FaceDetectedResult detect(InputStream imageInputStream) throws Exception { + Image img = ImageFactory.getInstance().fromInputStream(imageInputStream); + try (ZooModel model = criteria.loadModel(); + Predictor predictor = model.newPredictor()) { + DetectedObjects detection = predictor.predict(img); + return convertToFaceDetectedResult(detection,img); + /*saveBoundingBoxImage(img, detection); + return detection;*/ + } + } + + /** + * 转换检测结果 + * @param detection + * @param img + * @return + */ + private FaceDetectedResult convertToFaceDetectedResult(DetectedObjects detection, Image img){ + FaceDetectedResult faceDetectedResult = new FaceDetectedResult(); + List probabilities = new ArrayList<>(detection.getProbabilities()); + List detectedObjectList = detection.items(); + List RectangleList = detectedObjectList.parallelStream() + .map(obj -> { + Rectangle rectangle = new Rectangle(); + List pointList = new ArrayList<>(); + ai.djl.modality.cv.output.Rectangle rectangleDjl = obj.getBoundingBox().getBounds(); + int x = (int)(rectangleDjl.getX() * (double)img.getWidth()); + int y = (int)(rectangleDjl.getY() * (double)img.getHeight()); + int width = (int)(rectangleDjl.getWidth() * (double)img.getWidth()); + int height = (int)(rectangleDjl.getHeight() * (double)img.getHeight()); + pointList.add(new Point(x,y)); + pointList.add(new Point(x + width,y)); + pointList.add(new Point(x,y + height)); + pointList.add(new Point(x + width,y + height)); + rectangle.setPointList(pointList); + rectangle.setHeight(height); + rectangle.setWidth(width); + return rectangle; + }) + .collect(Collectors.toList()); + faceDetectedResult.setProbabilities(probabilities); + faceDetectedResult.setRectangles(RectangleList); + return faceDetectedResult; + } + + /** + * 特征提取 + * @param imagePath 图片路径 + * @return + * @throws Exception + */ + @Override + public float[] featureExtraction(String imagePath) throws Exception { + Path imageFile = Paths.get(imagePath); + Image img = ImageFactory.getInstance().fromFile(imageFile); + img.getWrappedImage(); + List mean = + Arrays.asList( + 127.5f / 255.0f, + 127.5f / 255.0f, + 127.5f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelUrls( + "https://resources.djl.ai/test-models/pytorch/face_feature.zip") + .optModelName("face_feature") // specify model file prefix + .optArgument("normalize", normalize) + .optTranslatorFactory(new ImageFeatureExtractorFactory()) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + + try (ZooModel model = criteria.loadModel()) { + Predictor predictor = model.newPredictor(); + return predictor.predict(img); + } + } + + /** + * 特征提取 + * @param inputStream 输入流 + * @return + * @throws Exception + */ + @Override + public float[] featureExtraction(InputStream inputStream) throws Exception { + Image img = ImageFactory.getInstance().fromInputStream(inputStream); + img.getWrappedImage(); + List mean = + Arrays.asList( + 127.5f / 255.0f, + 127.5f / 255.0f, + 127.5f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f, + 128.0f / 255.0f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelUrls( + "https://resources.djl.ai/test-models/pytorch/face_feature.zip") + .optModelName("face_feature") // specify model file prefix + .optArgument("normalize", normalize) + .optTranslatorFactory(new ImageFeatureExtractorFactory()) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + + try (ZooModel model = criteria.loadModel()) { + Predictor predictor = model.newPredictor(); + return predictor.predict(img); + } + } + + /** + * 计算相似度 + * @param feature1 图1特征 + * @param feature2 图2特征 + * @return + * @throws Exception + */ + @Override + public float calculSimilar(float[] feature1, float[] feature2) throws Exception { + float ret = 0.0f; + float mod1 = 0.0f; + float mod2 = 0.0f; + int length = feature1.length; + for (int i = 0; i < length; ++i) { + ret += feature1[i] * feature2[i]; + mod1 += feature1[i] * feature1[i]; + mod2 += feature2[i] * feature2[i]; + } + return (float) ((ret / Math.sqrt(mod1) / Math.sqrt(mod2) + 1) / 2.0f); + } + + /** + * 特征比较 + * @param imagePath1 图1路径 + * @param imagePath2 图2路径 + * @return + * @throws Exception + */ + @Override + public float featureComparison(String imagePath1, String imagePath2) throws Exception { + float[] feature1 = featureExtraction(imagePath1); + float[] feature2 = featureExtraction(imagePath2); + return calculSimilar(feature1, feature2); + } + + /** + * 特征比较 + * @param inputStream1 图1输入流 + * @param inputStream2 图2输入流 + * @return + * @throws Exception + */ + @Override + public float featureComparison(InputStream inputStream1, InputStream inputStream2) throws Exception { + float[] feature1 = featureExtraction(inputStream1); + float[] feature2 = featureExtraction(inputStream2); + return calculSimilar(feature1, feature2); + } + + /*@Override + public float[] recognize(FaceRegion region) { + return new float[0]; + }*/ + + /*@Override + public void loadModel(ModelConfig config) throws Exception { + + }*/ +}