【通用视觉】集成 OpenAI CLIP 模型，支持以图搜图、以文搜图、以图搜文等功能

【通用视觉】新增 YOLO 图像分类模型支持
【ASR/TTS】集成 Sherpa TTS（语音合成）与 ASR（语音识别）模块，支持中文、粤语、方言、英文等多种语言
【目标检测】优化视频目标检测功能

face/pom.xml

@@ -6,11 +6,11 @@
     <parent>
         <groupId>cn.smartjavaai</groupId>
         <artifactId>smartjavaai-parent</artifactId>
-        <version>1.0.25</version>
+        <version>1.0.26</version>
     </parent>
 
     <artifactId>face</artifactId>
-    <version>1.0.25</version>
+    <version>1.0.26</version>
     <name>face</name>
     <description>SmartJavaAI</description>
     <url>https://github.com/geekwenjie/SmartJavaAI</url>

face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java

@@ -1,13 +0,0 @@
-package cn.smartjavaai.face.enums;
-
-/**
- * @author dwj
- * @date 2025/5/31
- */
-public enum SimilarityType {
-
-    IP,     // 内积 (Inner Product)
-    L2,     // 欧氏距离 (Euclidean Distance)
-    COSINE  // 余弦相似度 (Cosine Similarity)
-
-}

face/src/main/java/cn/smartjavaai/face/model/facerec/CommonFaceRecModel.java

@@ -12,15 +12,16 @@ import cn.smartjavaai.common.cv.SmartImageFactory;
 import cn.smartjavaai.common.entity.*;
 import cn.smartjavaai.common.entity.face.FaceInfo;
 import cn.smartjavaai.common.entity.face.FaceSearchResult;
+import cn.smartjavaai.common.enums.SimilarityType;
 import cn.smartjavaai.common.pool.PredictorFactory;
 import cn.smartjavaai.common.utils.BufferedImageUtils;
 import cn.smartjavaai.common.utils.FileUtils;
 import cn.smartjavaai.common.utils.ImageUtils;
+import cn.smartjavaai.common.utils.SimilarityUtil;
 import cn.smartjavaai.face.config.FaceRecConfig;
 import cn.smartjavaai.face.constant.FaceDetectConstant;
 import cn.smartjavaai.face.entity.FaceRegisterInfo;
 import cn.smartjavaai.face.entity.FaceSearchParams;
-import cn.smartjavaai.face.enums.SimilarityType;
 import cn.smartjavaai.face.exception.FaceException;
 import cn.smartjavaai.face.factory.FaceRecModelFactory;
 import cn.smartjavaai.face.model.facerec.criteria.FaceRecCriteriaFactory;
@@ -808,6 +809,35 @@ public class CommonFaceRecModel implements FaceRecModel{
         return detectedResult;
     }
 
+    @Override
+    public R<float[]> extractFeatures(Image image, DetectionInfo detectionInfo) {
+        float[] features = null;
+        try (NDManager manager = model.getNDManager().newSubManager()) {
+            DJLImageFacePreprocessor djlImagePreprocessor = new DJLImageFacePreprocessor(image, manager);
+            DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
+            FaceInfo faceInfo = detectionInfo.getFaceInfo();
+            Image subImage = null;
+            //人脸对齐
+            if(config.isAlign()){
+                //人脸对齐
+                double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints());
+                djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112);
+                subImage = djlImagePreprocessor.process();
+            }else{
+                //裁剪
+                djlImagePreprocessor.enableCrop(rectangle);
+                if(config.isCropFace()){
+                    subImage = djlImagePreprocessor.process();
+                }
+            }
+            features = featureExtraction(subImage);
+            if (subImage != null && subImage.getWrappedImage() instanceof Mat) {
+                ((Mat)subImage.getWrappedImage()).release();
+            }
+        }
+        return Objects.isNull(features) ? R.fail(R.Status.Unknown) : R.ok(features);
+    }
+
     @Override
     public R<float[]> extractTopFaceFeature(Image image) {
         R<DetectionResponse> detectedResult = config.getDetectModel().detect(image);

face/src/main/java/cn/smartjavaai/face/model/facerec/FaceRecModel.java

@@ -2,6 +2,7 @@ package cn.smartjavaai.face.model.facerec;
 
 import ai.djl.inference.Predictor;
 import ai.djl.modality.cv.Image;
+import cn.smartjavaai.common.entity.DetectionInfo;
 import cn.smartjavaai.common.entity.DetectionResponse;
 import cn.smartjavaai.common.entity.R;
 import cn.smartjavaai.face.config.FaceRecConfig;
@@ -404,6 +405,10 @@ public interface FaceRecModel extends AutoCloseable{
         throw new UnsupportedOperationException("默认不支持该功能");
     }
 
+    default R<float[]> extractFeatures(Image image, DetectionInfo detectionInfo){
+        throw new UnsupportedOperationException("默认不支持该功能");
+    }
+
     /**
      * 特征提取（提取分数最高人脸特征）
      * 适用于单人脸场景

face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6FaceRecModel.java

@@ -2,12 +2,15 @@ package cn.smartjavaai.face.model.facerec;
 
 import ai.djl.engine.Engine;
 import ai.djl.modality.cv.Image;
+import ai.djl.ndarray.NDManager;
 import cn.smartjavaai.common.cv.SmartImageFactory;
 import cn.smartjavaai.common.entity.DetectionInfo;
+import cn.smartjavaai.common.entity.DetectionRectangle;
 import cn.smartjavaai.common.entity.DetectionResponse;
 import cn.smartjavaai.common.entity.R;
 import cn.smartjavaai.common.entity.face.FaceInfo;
 import cn.smartjavaai.common.enums.DeviceEnum;
+import cn.smartjavaai.common.enums.SimilarityType;
 import cn.smartjavaai.common.utils.BufferedImageUtils;
 import cn.smartjavaai.common.utils.FileUtils;
 import cn.smartjavaai.common.utils.ImageUtils;
@@ -17,10 +20,10 @@ import cn.smartjavaai.face.entity.FaceRegisterInfo;
 import cn.smartjavaai.face.entity.FaceResult;
 import cn.smartjavaai.face.entity.FaceSearchParams;
 import cn.smartjavaai.face.enums.FaceRecModelEnum;
-import cn.smartjavaai.face.enums.SimilarityType;
 import cn.smartjavaai.face.exception.FaceException;
 import cn.smartjavaai.face.factory.FaceDetModelFactory;
 import cn.smartjavaai.face.factory.FaceRecModelFactory;
+import cn.smartjavaai.face.preprocess.DJLImageFacePreprocessor;
 import cn.smartjavaai.face.utils.FaceUtils;
 import cn.smartjavaai.face.utils.Seetaface6Utils;
 import cn.smartjavaai.face.vector.config.MilvusConfig;
@@ -37,6 +40,7 @@ import io.milvus.param.MetricType;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.opencv.core.Mat;
 
 import javax.imageio.ImageIO;
 import java.awt.image.BufferedImage;
@@ -1086,6 +1090,42 @@ public class SeetaFace6FaceRecModel implements FaceRecModel{
         }
     }
 
+    @Override
+    public R<float[]> extractFeatures(Image image, DetectionInfo detectionInfo) {
+        FaceRecognizer faceRecognizer = null;
+        try {
+            SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
+            imageData.data = ImageUtils.getMatrixBGR(image);
+            faceRecognizer = faceRecognizerPool.borrowObject();
+
+            //提取特征
+            float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
+            FaceInfo faceInfo = detectionInfo.getFaceInfo();
+            if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
+                return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
+            }
+            SeetaPointF[] pointFS = Seetaface6Utils.convertToSeetaPointF(faceInfo.getKeyPoints());
+            //CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
+            boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
+            if(!isSuccess){
+                return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
+            }
+            return Objects.isNull(features) ? R.fail(R.Status.Unknown) : R.ok(features);
+        } catch (FaceException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new FaceException("人脸特征提取异常", e);
+        }finally {
+            if (faceRecognizer != null) {
+                try {
+                    faceRecognizerPool.returnObject(faceRecognizer); //归还
+                } catch (Exception e) {
+                    log.warn("归还Predictor失败", e);
+                }
+            }
+        }
+    }
+
     @Override
     public R<float[]> extractTopFaceFeature(Image image) {
         float[] features = null;

face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java

@@ -1,129 +0,0 @@
-package cn.smartjavaai.face.utils;
-
-import cn.smartjavaai.face.enums.SimilarityType;
-
-/**
- * 特征相似度计算工具类
- * 支持三种计算方式：IP（内积）、L2（欧氏距离）、COSINE（余弦相似度）
- * 所有计算结果归一化到[0,1]范围
- */
-public class SimilarityUtil {
-
-    /**
-     * 计算特征相似度
-     * @param features1 特征向量1
-     * @param features2 特征向量2
-     * @param similarityType 计算类型 (IP, L2, COSINE)
-     * @param normalizeScore 是否归一化结果到 [0,1]
-     * @return 相似度
-     */
-    public static float calculate(float[] features1, float[] features2,
-                                  SimilarityType similarityType,
-                                  boolean normalizeScore) {
-        validateInput(features1, features2);
-
-        switch (similarityType) {
-            case IP:
-                return innerProductSimilarity(features1, features2, normalizeScore);
-            case L2:
-                return euclideanSimilarity(features1, features2, normalizeScore);
-            case COSINE:
-                return cosineSimilarity(features1, features2, normalizeScore);
-            default:
-                throw new IllegalArgumentException("不支持的相似度计算类型: " + similarityType);
-        }
-    }
-
-    // ================ 私有计算方法 ================
-
-    /**
-     * 计算内积相似度（归一化到[0,1]）
-     * 适用于归一化向量（结果范围[-1,1] -> [0,1]）
-     */
-    private static float innerProductSimilarity(float[] v1, float[] v2, boolean normalize) {
-        float dot = dotProduct(v1, v2);
-        return normalize ? (dot + 1.0f) / 2.0f : dot;
-    }
-
-    /**
-     * 计算欧氏距离相似度（归一化到[0,1]）
-     * 距离越小相似度越高，距离为0时相似度为1
-     */
-    private static float euclideanSimilarity(float[] v1, float[] v2, boolean normalize) {
-        float dist = euclideanDistance(v1, v2);
-        return normalize ? 1.0f / (1.0f + dist) : dist;
-    }
-
-    /**
-     * 计算余弦相似度（归一化到[0,1]）
-     * 适用于非归一化向量（结果范围[-1,1] -> [0,1]）
-     */
-    private static float cosineSimilarity(float[] v1, float[] v2, boolean normalize) {
-        float dot = dotProduct(v1, v2);
-        float norm1 = vectorNorm(v1);
-        float norm2 = vectorNorm(v2);
-
-        if (norm1 <= 0 || norm2 <= 0) {
-            return 0.0f;
-        }
-
-        float cosine = dot / (norm1 * norm2);
-        return normalize ? (cosine + 1.0f) / 2.0f : cosine;
-    }
-
-    // ================ 基础向量操作 ================
-
-    /**
-     * 计算点积（内积）
-     */
-    public static float dotProduct(float[] v1, float[] v2) {
-        float sum = 0.0f;
-        for (int i = 0; i < v1.length; i++) {
-            sum += v1[i] * v2[i];
-        }
-        return sum;
-    }
-
-    /**
-     * 计算欧氏距离
-     */
-    public static float euclideanDistance(float[] v1, float[] v2) {
-        float sumSquaredDiff = 0.0f;
-        for (int i = 0; i < v1.length; i++) {
-            float diff = v1[i] - v2[i];
-            sumSquaredDiff += diff * diff;
-        }
-        return (float) Math.sqrt(sumSquaredDiff);
-    }
-
-    /**
-     * 计算向量模长
-     */
-    public static float vectorNorm(float[] vector) {
-        float sum = 0.0f;
-        for (float v : vector) {
-            sum += v * v;
-        }
-        return (float) Math.sqrt(sum);
-    }
-
-    // ================ 输入验证 ================
-
-    /**
-     * 验证输入向量
-     */
-    private static void validateInput(float[] v1, float[] v2) {
-        if (v1 == null || v2 == null) {
-            throw new IllegalArgumentException("特征向量不能为null");
-        }
-
-        if (v1.length == 0 || v2.length == 0) {
-            throw new IllegalArgumentException("特征向量不能为空");
-        }
-
-        if (v1.length != v2.length) {
-            throw new IllegalArgumentException("特征向量长度不一致: " +
-                    v1.length + " vs " + v2.length);
-        }
-    }
-}

face/src/main/java/cn/smartjavaai/face/vector/config/SQLiteConfig.java

@@ -1,6 +1,6 @@
 package cn.smartjavaai.face.vector.config;
 
-import cn.smartjavaai.face.enums.SimilarityType;
+import cn.smartjavaai.common.enums.SimilarityType;
 import cn.smartjavaai.face.enums.VectorDBType;
 import lombok.Data;
 import lombok.EqualsAndHashCode;

face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java

@@ -2,9 +2,9 @@ package cn.smartjavaai.face.vector.core;
 
 import cn.hutool.core.util.IdUtil;
 import cn.smartjavaai.common.config.Config;
+import cn.smartjavaai.common.utils.SimilarityUtil;
 import cn.smartjavaai.face.dao.FaceDao;
 import cn.smartjavaai.face.entity.FaceSearchParams;
-import cn.smartjavaai.face.utils.SimilarityUtil;
 import cn.smartjavaai.face.vector.config.SQLiteConfig;
 import cn.smartjavaai.face.vector.entity.FaceVector;
 import cn.smartjavaai.common.entity.face.FaceSearchResult;