diff --git a/README.md b/README.md index 12a273a..c718f5f 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸检测

+

人脸检测(Face Detection)

- 5点人脸关键点定位
@@ -59,7 +59,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸比对1：1

+

人脸比对1：1
(Face Recognition)

- 人脸对齐
@@ -72,7 +72,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人证核验

+

人证核验
(Face ID Verification)

@@ -84,7 +84,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸比对1：N

+

人脸比对1：N
(Face Recognition)

- 人脸对齐
- 人脸注册
- 人脸库查询
@@ -101,11 +101,11 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的

人脸属性检测

- - 性别检测
- - 年龄检测
- - 口罩检测
- - 眼睛状态检测
- - 脸部姿态检测
+ - 性别检测(GenderDetection)
+ - 年龄检测(AgeDetection)
+ - 口罩检测(Face Mask Detection)
+ - 眼睛状态检测(EyeClosenessDetection)
+ - 脸部姿态检测(FacePoseEstimation)
@@ -117,7 +117,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

活体检测

+

活体检测
(FaceLivenessDetection)

- 图片和视频活体检测
@@ -130,7 +130,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸表情识别

+

人脸表情识别
(FacialExpressionRecognition)

- 7种表情检测
@@ -140,10 +140,23 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
+ + +
+

图像分类
(Image Classification)

+
+ + +
+ +
+ +
-

目标检测

+

目标检测(Object Detection)

+ - 视频流目标检测：rtsp、摄像头、视频文件等
@@ -151,11 +164,40 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 + +
+ +
+ - +
-

自定义目标训练+检测

+

语义分割
(Semantic Segmentation)

+
+ + +
+ +
+ + + + +
+

实例分割
(Instance Segmentation)

+
+ + +
+ +
+ + + + +
+

自定义目标训练+检测
(Object Detection)

@@ -167,7 +209,55 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

OCR文字识别

+

行人检测(Person Detection)

+
+ + +
+ +
+ + + + +
+

人类动作识别
(Action Recognition)

+
+ + +
+ +
+ + + + +
+

OBB旋转框检测
(Oriented Bounding Boxes Object Detection)

+
+ + +
+ +
+ + + + +
+

姿态估计
(Pose Estimation)

+
+ + +
+ +
+ + + + +
+

OCR文字识别
(Text Recognition)

- 支持任意角度文字识别
- 支持印刷体识别
- 支持手写字识别
@@ -183,7 +273,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的

OCR文字识别

- - 表格识别
+ - 表格识别
(Table Structure Recognition)
@@ -195,7 +285,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

车牌识别

+

车牌识别
(License Plate Recognition)

- 单层/双层检测
- 车牌颜色识别
@@ -205,7 +295,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
- +
@@ -214,7 +304,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

机器翻译

+

机器翻译
(Machine Translation)

- 200多种语言互相翻译
@@ -224,47 +314,102 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 + + +
+

语音识别
(Speech Recognition)

+ - 支持100种语言
+ - 支持实时语音识别 +
+ + +
+ +
+ + + + +
+

语音合成(ASR)

+
+ + +
+ +
+ + + + +
+

CLIP

+ - 文搜图
+ - 图搜图
+ - 图搜文
+
+ + +
+ +
+ + -## 🚀 核心亮点 +## 🚀 SmartJavaAI 解决了什么问题？ 针对 Java 开发者在集成智能算法时常见的两大痛点： - 🐍 主流AI深度学习框架（PyTorch/TensorFlow）的Python生态与Java工程体系割裂 -- ⚙️ 现有算法方案分散杂乱，封装不统一，使用门槛高，不易直接服务于 Java 业务开发 +- ⚙️ 现有AI集成方案分散杂乱，封装不统一，使用门槛高，不易直接服务于 Java 业务开发 我们实现了： -✅ **开箱即用** - 两行代码即可调用算法 +✅ **开箱即用** - 两行代码即可调用算法 ✅ **支持多种深度学习引擎** - Pytorch、Tensorflow、MXNet、ONNX Runtime -✅ **功能丰富** - 当前支持人脸识别与目标检测，未来将陆续支持 OCR、图像分类、NLP 等多个 AI 领域任务，构建全面的智能算法体系。 +✅ **功能丰富** - 当前支持人脸识别、目标检测、OCR、NLP 等多个 AI 领域任务，构建全面的智能算法体系。 -✅ **跨平台兼容** - 支持Windows/Linux/macOS系统（x86 & ARM架构） +✅ **跨平台兼容** - 支持Windows/Linux/macOS系统（x86 & ARM架构） ## 📌 支持功能 - -### ✅ 已实现功能 - -- **人脸识别** - - 人脸检测：5点人脸关键点定位 +- **人脸识别** + - 人脸检测：人脸区域检测、5点人脸关键点定位 - 人脸识别：人脸512维特征提取、人脸对齐、1:1 人脸比对、1:N 人脸识别 - 人脸库：人脸注册、更新、查询、删除(支持向量数据库milvus/sqlite) - 人脸属性检测：性别、年龄、口罩、眼睛状态、脸部姿态 - 静默活体检测：图片、视频活体检测 - 人脸表情识别：7种表情识别 - 人脸质量评估：亮度评估、清晰度评估、完整度评估、姿态评估、遮挡评估 +- **图像分类** + - 支持多种主流模型：集成 YOLOv8、YOLOv11 等分类模型 + - 支持自定义模型加载：可无缝加载并部署用户自行训练的分类模型 - **目标检测** - - 支持多种主流模型：兼容 YOLOv3、YOLOv5、YOLOv8、YOLOv11、YOLOv12、SSD 等目标检测算法 + - 支持多种主流模型：集成 YOLOv5、YOLOv8、YOLOv11、YOLOv12、Tensorflow Object Detection 等目标检测模型 - 支持自定义模型加载：可无缝加载并部署用户自行训练的目标检测模型 + - 集成行人检测模型 +- **语义分割** + - 集成DeepLabV3模型 +- **实例分割** + - 集成YOLOv8-seg、YOLOv11-seg、Mask R-CNN等模型 +- **OBB旋转框目标检测** + - 集成YOLOv11-obb模型 +- **动作识别** + - 支持KINETICS400数据集中400个人类动作识别 +- **姿态估计** + - 集成YOLOv8-pose、YOLOv11-pose等模型 +- **CLIP** + - 支持提取图片及文本特征 + - 支持文搜图、图搜文、图搜图 - **OCR文字识别** - 支持PaddleOCR 3.0模型：集成最新PP-OCRv5、PP-OCRv4、表格结构识别模型(SLANet_plus)、文本行方向分类模型 - 支持任意角度识别，方向校准 @@ -273,33 +418,41 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 支持中文车牌识别：单层/双层检测，颜色识别，支持12种中文车牌 - **机器翻译** - 集成NLLB-200模型：支持200+语言互相翻译 +- **语音识别(ASR)** + - 集成openai的whisper模型：支持100种语言 + - 集成vosk语音识别 + - 集成sherpa-onnx语音识别 +- **语音合成(TTS)** + - 集成sherpa-onnx：支持中文、方言、粤语、英文、德语等多种语言 ## 🌟 AI集成方式对比 -| 方案 | 技术特点 | 优点 | 缺点 | -|---------------------|--------------------------------------------------------------------------|---------------------------------------------------------------------|-----------------------------------------------| -| **OpenCV** | 传统图像处理方案 | ✅ 提供java接口
✅ 轻量级部署
✅ 社区资源丰富 | ❌ 基于传统算法精度低(60%-75%)
❌ 需本地安装环境 | +| 方案 | 技术特点 | 优点 | 缺点 | +|---------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------|-----------------------------------------------| +| **OpenCV** | 传统图像处理方案 | ✅ 提供java接口
✅ 轻量级部署
✅ 社区资源丰富 | ❌ 基于传统算法精度低(60%-75%)
❌ 需本地安装环境 | | ‌**商业闭源SDK（如虹软等）**‌ | 商业级闭源解决方案 | ✅ 开箱即用
✅ 提供完整文档和SDK
✅ 支持离线活体检测 | ❌ 免费版需年度授权更新
❌ 商业授权费用高
❌ 代码不可控 | | ‌**云API(阿里云)**‌ | SaaS化云端服务 | ✅ 零部署成本
✅ 支持高并发
✅ 自带模型迭代 | ❌ 网络延迟风险(200-800ms)
❌ 按调用量计费
❌ 有数据安全风险 | | ‌**Python混合调用**‌ | 跨语言调用方案 | ✅ 可集成PyTorch/TF等框架
✅ 支持自定义算法
✅ 识别精度高 | ❌ 需维护双语言环境
❌ 进程通信性能损耗(30%+)
❌ 异常处理复杂度翻倍 | -| ‌**JNI/JNA**‌ | 跨语言底层调用方案 | ✅ 直接调用 C/C++ 高性能算法库✅ 支持调用各种原生成熟库✅ 可封装成通用工具Jar | ❌ 开发成本高，JNI更复杂❌ 跨平台兼容性差 | +| ‌**JNI/JNA**‌ | 跨语言底层调用方案 | ✅ 直接调用 C/C++ 高性能算法库
✅ 支持调用各种原生成熟库
✅ 可封装成通用工具Jar | ❌ 开发成本高，JNI更复杂
❌ 跨平台兼容性差 | | ‌**DJL框架**‌ | 深度学习框架 | ✅ 纯Java实现
✅ 支持主流深度学习框架
✅ 可加载预训练模型(99%+) | ❌ 需掌握DL知识
❌ 需处理模型加载、预处理、后处理等复杂技术细节 | -| ‌**SmartJavaAI**‌ | java深度学习工具包 | ✅ 支持主流深度学习框架
✅ 提供丰富、开箱即用API
✅ 上手简单，单一Jar包集成 | 无 | +| ‌**SmartJavaAI**‌ | java深度学习工具包 | ✅ 支持主流深度学习框架
✅ 提供丰富、开箱即用API
✅ 上手简单，单一Jar包集成 | 无 | ## 🛠️包含组件 -| 模块 | 介绍 | -|-----------------------------|---------------------------| -| smartjavaai-common | 基础通用模块，封装了公共功能，供各算法模块共享使用 | -| smartjavaai-face | 人脸功能模块 | -| smartjavaai-objectdetection | 目标检测模块 | -| smartjavaai-ocr | OCR文字识别模块 | -| smartjavaai-translate | 机器翻译模块 | +| 模块 | 介绍 | +|-----------------------|---------------------------| +| common | 基础通用模块，封装了公共功能，供各算法模块共享使用 | +| bom | 依赖管理模块 | +| face | 人脸功能模块 | +| vision | 通用视觉模块(目标检测等功能) | +| ocr | OCR文字识别模块 | +| translate | 机器翻译模块 | +| speech | 语音功能模块，包含 ASR 和 TTS | -可以根据需求对每个模块单独引入，也可以通过引入`smartjavaai-all`方式引入所有模块。 +可以根据需求对每个模块单独引入，也可以通过引入`all`方式引入所有模块。 ------------------------------------------------------------------------------- @@ -316,22 +469,52 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ### 1、环境要求 - Java 版本：**JDK 8或更高版本** -- 操作系统：不同模型支持的系统不一样，具体请查看文档 +- 操作系统：不同模型支持的系统不一样，具体请查看[文档](http://doc.smartjavaai.cn) ### 2、Maven -在项目的pom.xml的dependencies中加入以下内容（全部功能），也可以根据需求对每个模块单独引入: + +在项目的 `pom.xml` 的 `dependencies` 中可以一次性引入全部功能（如下所示）。 + +⚠️ **注意：不推荐直接引入全部依赖**，更推荐根据实际需求，按功能模块单独引入，避免引入不必要的包。 + +详细引入方式请查看 [文档](http://doc.smartjavaai.cn/install.html)、或查看[示例代码](https://gitee.com/dengwenjie/SmartJavaAI/tree/master/examples) ```xml cn.smartjavaai - smartjavaai-all - 1.0.22 + all + 1.0.27 ``` + ### 3、完整示例代码 [示例代码](https://gitee.com/dengwenjie/SmartJavaAI/tree/master/examples) +本项目在 `examples` 文件夹下提供了多个示例工程，用于演示各功能模块的使用方法： + +* `face-example`：人脸检测、人脸识别等功能示例 +* `vision-example`：通用视觉检测示例：目标检测、目标分割、图像分类等 +* `ocr-example`：OCR文字识别、车牌识别等功能示例 +* `translate-example`：机器翻译功能示例 +* `speech-example`：语音识别、语音合成功能示例 + +#### 运行方式 + +如果你只想运行某个示例，请按以下方式操作： + +1. 打开 IDEA（或你喜欢的 IDE） +2. 选择 **“Open”**，然后仅导入 `examples` 目录下对应的示例项目，例如： + + ``` + examples/face-example + ``` +3. IDEA 会自动识别并加载依赖。若首次导入，请等待 Maven 下载依赖完成。 +4. 请从我们提供的 百度网盘 中下载模型及其附带文件，并在示例代码中将模型路径修改为您本地的实际路径。 +5. 可通过查看每个 Java 文件顶部的注释了解对应功能，或参考 README 文件中对各 Java 文件功能的说明，运行相应的测试方法进行体验。 + + + ### 4、文档地址 [开发文档](http://doc.smartjavaai.cn) @@ -344,39 +527,37 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 **人脸检测模型(FaceDetection、FaceLandmarkExtraction)** -支持功能： -- 人脸检测 -- 5点人脸关键点定位 - -| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | -| ----------- |---------|-------------------------|----------------------------------------------------------------------------------------| -| RetinaFace | PyTorch | 高精度人脸检测模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | -| UltraLightFastGenericFace | PyTorch | 针对边缘计算设备设计的轻量人脸检测模型 | [Github](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | -| SeetaFace6 | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|------------------------------|-------------|------------------------------|---------------------------------------------------------------------------------| +| MTCNN(均衡) | PyTorch | 使用最广泛的模型之一，经典多阶段人脸检测，速度与精度均衡 | [Github](https://github.com/timesler/facenet-pytorch) | +| SeetaFace6(均衡模型) | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| RetinaFace(高精度) | PyTorch | 基于单阶段检测的多尺度人脸检测 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| RetinaFace_1080x720(高精度) | OnnxRuntime | 针对高分辨率图像优化的 RetinaFace 模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| RetinaFace_640x640(高精度) | OnnxRuntime | 针对低分辨率图像优化的 RetinaFace 模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| yolov5face_n_0.5_320x320(极速) | OnnxRuntime | YOLOv5 小型人脸检测模型，设计轻量，适合快速推理 | 无 | +| yolov5face_m_640x640(极速) | OnnxRuntime | YOLOv5 中型人脸检测模型 | 无 | +| UltraLightFastGenericFace(极速) | PyTorch | 针对边缘计算设备设计的轻量人脸检测模型 | [Github](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | --- **人脸识别模型(FaceRecognition)** -支持功能： -- 人脸512维特征提取 -- 人脸对齐(人脸矫正) -- 人脸特征比对（内积[IP]、欧氏距离[L2]、余弦相似度[COSINE]） -| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | -|-------|---------|---------------------------------------------------------|----------------------------------------------------------------------------------------| -| InsightFace_IR-SE50| PyTorch | （高精度）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | -| InsightFace_Mobilefacenet | PyTorch | （轻量级）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | -| FaceNet | PyTorch | 基于 PyTorch 的 Inception ResNet（V1）模型仓库 | [Github](https://github.com/timesler/facenet-pytorch) | -| ElasticFace| PyTorch | 基于 CVPRW2022 论文《ElasticFace: Elastic Margin Loss for Deep Face Recognition》实现的人脸识别模型| [Github](https://github.com/fdbtrs/ElasticFace) | -| SeetaFace6 | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------------------------|---------------------|--------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| +| InsightFace_IR-SE50(高精度) | PyTorch | 这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | +| InsightFace_Mobilefacenet(极速) | PyTorch | （轻量级）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | +| FaceNet | PyTorch | 基于 PyTorch 的 Inception ResNet（V1）模型仓库 | [Github](https://github.com/timesler/facenet-pytorch) | +| ElasticFace | PyTorch | 基于 CVPRW2022 论文《ElasticFace: Elastic Margin Loss for Deep Face Recognition》实现的人脸识别模型 | [Github](https://github.com/fdbtrs/ElasticFace) | +| SeetaFace6 | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| SphereFace-20A | PyTorch+OnnxRuntime | SphereFace:Deep Hypersphere Embedding for Face Recognition | [Github](https://github.com/wy1iu/sphereface) | +| DREAM_IJBA_RES18 | OnnxRuntime | DREAM block for Pose-Robust Face Recognition | [Github](https://github.com/penincillin/DREAM) | +| face.evoLVe | PyTorch | High-Performance Face Recognition Library based on PaddlePaddle & PyTorch | [Github](https://github.com/ZhaoJ9014/face.evoLVe) | +| VGGFace2 | PyTorch | PyTorch Face Recognizer based on 'VGGFace2: A dataset for recognising faces across pose and age'. | [Github](https://github.com/cydonia999/VGGFace2-pytorch) | **静态活体检测(RGB)模型（Silent face-anti-spoofing、FaceLivenessDetection）** -支持功能： -- 检测图片中的人脸是否为来自认证设备端的近距离裸拍活体人脸对象(裸拍活体正面人脸是指真人未经重度PS、风格化、人工合成等后处理的含正面人脸) - | 模型名称 | 引擎 | 模型简介 | 模型开源网站 | |-----------|----------------|-------------------------|----------------------------------------------------------------------| | MiniVision| OnnxRuntime | 小视科技的静默活体检测 | [Github](https://github.com/minivision-ai/Silent-Face-Anti-Spoofing) | @@ -425,24 +606,36 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 --- -#### 目标检测模型 - -支持功能： -- 自训练模型推理 -- yolov3~yolov12 系列 +#### 图像分类模型(CLS) **YOLO 系列** -|模型名称 | 引擎 | 版本 | 大小（Backbone） |
数据集
|
精度
|
速度
|
适用场景
| -| :---------- |-------------| -------| ----------- |----------------------------------------|-----------------------------------|-------|--------------------------------------| -| YOLO12N | OnnxRuntime | v12 | 极轻量 | COCO | 高 | 极快 | YOLO 系列最新版本，精度与速度进一步优化，适合高实时性要求场景 | -|YOLO11N | PyTorch | v11 | 极轻量 | COCO | 中等偏上 | 极快 | 与 v8n 类似，版本更新点在兼容性与 API | -|YOLOV8N | PyTorch | v8 | 极轻量 | COCO | 中等偏上 | 极快 | 对实时性要求极高的应用 | -|YOLOV5S | PyTorch | v5 | 小型 | COCO | 较高 | 非常快 | 常见通用场景，算力资源有限时优选 | -|YOLOV5S_ONNXRUNTIME| OnnxRuntime | v5 | 小型 | COCO | 较高 | 加速（需 ONNX 支持） | Windows/Linux 通用加速部署 | -|YOLO (MXNet / 通用模型) | MXNet | v3 | DarkNet‑53 | COCO | 较高 | 快 | 需要 MXNet 生态或复现老项目时使用 | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|----------------|------------|----------------------------------------------------------------------| +| YOLOV11-cls | OnnxRuntime | 最流行的图像分类模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV8-cls | OnnxRuntime | 最流行的图像分类模型 | [Github](https://docs.ultralytics.com/zh/models/) | +#### 目标检测模型 + +**YOLO 系列** + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|----------------|-------------------------|----------------------------------------------------------------------| +| YOLOV12 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV11 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV8 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | + +**Tensorflow系列** + +仅测试了以下Tensorflow目标检测模型 + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|-------------|----------------|----------------------------------------------------------------------| +| EfficientDet | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | +| SSD MobileNet V2 | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | +| Faster RCNN Inception Resnet V2 | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | + **SSD 系列** | 模型名称 | 引擎 | 骨干网络 | 输入尺寸 |
训练数据集
| 精度（mAP） |
推理速度
|
适用场景
| @@ -453,21 +646,68 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 |SSD_300_VGG16_ATROUS_VOC| MXNet | VGG‑16 | 300×300 | Pascal VOC | 中等偏上 | 快 | VOC 数据集同类任务；资源受限时使用| |SSD_512_MOBILENET1_VOC| MXNet | MobileNet‑1.0 | 512×512 | Pascal VOC | 中等 | 快 | 嵌入式/移动端设备；算力和内存都很有限| +--- -**YOLOv3 变体系列** +#### 语义分割模型 -|模型名称| 引擎 | 骨干网络 |
数据集
| 输入尺寸 |
精度
|
速度
|
适用场景
| -| :-----|---------- | -------| ----------- |------------| ---------|-------|--------------------------------------| -|YOLO3_DARKNET_VOC_416|MXNet | DarkNet‑53 | VOC | 416×416 | 高 | 中等 | VOC 任务复现；精度优先 | -|YOLO3_DARKNET_COCO_320 |MXNet| DarkNet‑53 | COCO | 320×320 | 中等 | 快 | COCO 小模型测试；资源受限 | -|YOLO3_DARKNET_COCO_416 |MXNet| DarkNet‑53 | COCO | 416×416 | 高 | 中等 | 通用 COCO 部署；精度优先 | -|YOLO3_DARKNET_COCO_608 |MXNet| DarkNet‑53 | COCO | 608×608 | 很高 | 慢| 批量离线推理；精度要求极高 | -|YOLO3_MOBILENET_VOC_320 |MXNet| MobileNet‑V1 | VOC | 320×320 | 中等 | 非常快| 嵌入式设备；VOC 小目标任务 | -|YOLO3_MOBILENET_VOC_416 |MXNet| MobileNet‑V1 | VOC | 416×416 | 高 | 快| 移动端 VOC 部署 | -|YOLO3_MOBILENET_COCO_320 |MXNet| MobileNet‑V1 | COCO | 320×320 | 中等 | 非常快 | 嵌入式设备；COCO 小目标任务 | -|YOLO3_MOBILENET_COCO_416 |MXNet| MobileNet‑V1 | COCO | 416×416 | 高 | 快 | 移动端 COCO 部署 | -|YOLO3_MOBILENET_COCO_608 |MXNet| MobileNet‑V1 | COCO | 608×608 | 很高 | 中等 | 对精度要求较高的移动端任务 | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|---------|-------------------------|--------| +| DeepLabv3 | PyTorch | DeepLabv3 是一种图像分割模型，可以把图片里的每个像素分类出来 | 无 | +--- + +#### 实例分割模型 + + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|---------|------------------------------------|------------| +| YOLOV8-SEG | OnnxRuntime | Ultralytics在COCO 数据集 上训练的模型 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +| YOLOV11-SEG | OnnxRuntime | Ultralytics在COCO 数据集 上训练的模型 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +| Mask R-CNN | MXNet | Mask R-CNN 是一种在目标检测基础上，同时为每个物体生成像素级分割区域的深度学习模型 | 无 | + + +#### CLIP模型(Connecting text and images) + + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|-------------|--------------------|------------| +| clip-vit-base-patch32 | PyTorch | openai的图片及文本特征提取模型 | [Github](https://github.com/openai/CLIP) | + +--- + +#### OBB旋转框目标检测模型 + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|---------|--------------------------------|------------| +| YOLOV11-OBB | OnnxRuntime | Ultralytics在DOTAv1 数据集 上训练的模型、通过引入一个额外的角度来更准确地定位图像中的对象 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +--- + +#### 行人检测模型 + +| 模型名称 | 引擎 | 模型开源网站 | +|-------------|---------|-----| +| YOLOV8_PERSON | OnnxRuntime | [Github](https://github.com/YapaLab/yolo-face) | + +--- + +#### 人类动作识别模型 + +| 模型名称 | 引擎 | +|-------------|---------| +| VIT_BASE_PATCH16 | PyTorch | +| INCEPTIONV3_KINETICS400 | OnnxRuntime | +| INCEPTIONV1_KINETICS400 | OnnxRuntime | +| RESNET_V1B_KINETICS400 | OnnxRuntime | +--- + +#### 姿态估计模型 + + +| 模型名称 | 引擎 | 模型开源网站 | +|--------------|-------------|----------------------------------------------------------| +| YOLO11N-POSE | OnnxRuntime | [Github](https://docs.ultralytics.com/zh/tasks/pose/) | +| YOLO8N-POSE | OnnxRuntime | [Github](https://docs.ultralytics.com/zh/tasks/pose/) | +| SIMPLE_POSE | MXNet | 无 | --- #### OCR 模型 @@ -536,9 +776,32 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 支持功能 - 200多语言互相翻译 -| 模型名称 | 模型简介 | 模型官网 | -| ------------------------------ | ------------------------ |-----------------------------------------------| -| NLLB-200 | Meta AI 开发的一个先进的单一多语言机器翻译模型 | [Github](https://github.com/facebookresearch/fairseq/tree/nllb) | +| 模型名称 | 模型简介 | 模型官网 | +|---------------|-----------------------------------------|-----------------------------------------------| +| NLLB-200 | Meta AI 开发的一个先进的单一多语言机器翻译模型 | [Github](https://github.com/facebookresearch/fairseq/tree/nllb) | +| OPUS_MT_ZH_EN | Helsinki-NLP团队开发的中英翻译模型，基于Transformer架构 | [huggingface](https://huggingface.co/Helsinki-NLP/opus-mt-zh-en) | +| OPUS_MT_EN_ZH | Helsinki-NLP团队开发的英中翻译模型，基于Transformer架构 | [huggingface](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) | + +--- + +#### 语音识别模型(ASR) + +这里仅介绍模型的开源项目，每个开源项目通常包含多个具体模型，本文不逐一列出。 + +| 模型名称 | 模型简介 | 模型官网 | +|---------|-------------------------------------------------------------------------|-----------------------------------------------| +| Whisper | OpenAI 开源的通用语音识别（ASR）模型，支持多语言转写和翻译，具有较高的识别精度，尤其在嘈杂环境中表现良好，适合离线和批量音频处理。 | [Github](https://github.com/ggml-org/whisper.cpp) | +| Vosk | 一个轻量级离线语音识别工具包，支持多种语言和平台（包括移动端与嵌入式设备），可在低资源环境中运行，适合实时语音识别场景。 | [Github](https://github.com/alphacep/vosk-api) | +| sherpa-onnx | Sherpa-ONNX 是一个基于 ONNX Runtime 的ASR 及 TTS 推理框架 | [Github](https://github.com/k2-fsa/sherpa-onnx) | + +#### 语音合成模型(TTS) + +这里仅介绍模型的开源项目，每个开源项目通常包含多个具体模型，本文不逐一列出。 + +| 模型名称 | 模型简介 | 模型官网 | +|---------|-------------------------------------------------------------------------|-----------------------------------------------| +| sherpa-onnx | Sherpa-ONNX 是一个基于 ONNX Runtime 的ASR 及 TTS 推理框架 | [Github](https://github.com/k2-fsa/sherpa-onnx) | + --- @@ -551,7 +814,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 联系方式 -如您在使用过程中有任何问题或建议，欢迎添加微信，与我们交流并加入用户交流群 +如您在使用过程中有任何问题、建议，或希望进行技术交流与合作，欢迎添加微信与我联系，并加入用户交流群。 - **微信**: deng775747758 （请备注：SmartJavaAI） - **Email**: 775747758@qq.com @@ -559,52 +822,19 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 🚀 **如果这个项目对你有帮助，别忘了点个 Star ⭐！你的支持是我持续优化升级的动力！** ❤️ +## 献代码的步骤 -## 近期更新日志 +1、在Gitee或者Github/Gitcode上fork项目到自己的repo -## [v1.0.22] - 2025-07-28 -- 新增 Milvus 身份验证支持 -- 集成车牌识别模型，支持车牌检测与识别 -- 目标检测功能升级：可指定类别及topk -- 支持自定义线程池线程数量 +2、把fork过去的项目也就是你的项目clone到你的本地 + +3、修改代码（记得一定要修改dev分支） + +4、commit后push到自己的库（dev分支） + +5、登录Gitee或Github/Gitcode在你首页可以看到一个 pull request 按钮，点击它，填写一些说明信息，然后提交即可。 + +6、等待维护者合并 -## [v1.0.20] - 2025-07-18 -- OCR：新增表格识别模型 -- OCR：新增9个通用模型 -- OCR：支持批量检测识别 -- OCR：新增更多参数，使用更加灵活 -- 人脸识别：支持ID查询及分页获取人脸信息 -- 活体检测：视频检测支持设置最大帧数 - -## [v1.0.19] - 2025-07-06 -- 人脸模块：新增小视科技（MiniVision）活体检测模型 -- 人脸模块：新增阿里通义工作室活体检测模型 -- 人脸模块：新增 2 个表情识别模型 -- 人脸模块：新增 InsightFace 和 ElasticFace 人脸识别模型 -- 人脸模块：新增 Seetaface6 质量评估模型 -- 目标检测模块：支持更多自定义模型参数配置 -- 人脸模块：支持 Base64 编码图片输入 -- 通用功能：实现 AutoCloseable 接口，支持资源自动释放 -- OCR 模块：修复加方向矫正后无法连续识别的问题 -- 人脸模块：修复人脸更新后的缓存异常问题 -- 其他：优化部分功能与细节体验 - -## [v1.0.17] - 2025-06-18 -- 新增机器翻译模块：支持 200+ 种语言之间的相互翻译 -- 人脸识别模块：修复批量删除人脸数据时的异常问题 -- 人脸识别模块：修复人脸检索 Top大于 1 时报异常问题 - -## [v1.0.16] - 2025-06-09 -- 人脸模块：人脸查询支持 Milvus 和 SQLite -- 人脸模块：FaceNet人脸模型也支持人脸注册，查询等功能 -- 人脸模块：Seetaface6 自动下载人脸库 -- 人脸模块：Seetaface6解决依赖库重复下载问题 -- 人脸模块：支持手动加载人脸库 -- 人脸模块：人脸识别相关功能支持更多参数 - - -## [v1.0.15] - 2025-05-17 -- 新增OCR文字识别模块：支持最新 PP-OCRv5 -- OCR文本识别：支持文字方向检测与自动校正 diff --git a/smartjavaai-all/pom.xml b/all/pom.xml similarity index 93% rename from smartjavaai-all/pom.xml rename to all/pom.xml index e9c35f1..be65c94 100644 --- a/smartjavaai-all/pom.xml +++ b/all/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.22 + 1.0.27 - smartjavaai-all - 1.0.22 + all + 1.0.27 ${project.artifactId} SmartJavaAI https://github.com/geekwenjie/SmartJavaAI @@ -33,25 +33,31 @@ cn.smartjavaai - smartjavaai-face + face ${project.version} cn.smartjavaai - smartjavaai-objectdetection + vision ${project.version} cn.smartjavaai - smartjavaai-ocr + ocr ${project.version} cn.smartjavaai - smartjavaai-translate + translate + ${project.version} + + + + cn.smartjavaai + speech ${project.version} diff --git a/all/src/test/java/Test.java b/all/src/test/java/Test.java new file mode 100644 index 0000000..245980a --- /dev/null +++ b/all/src/test/java/Test.java @@ -0,0 +1,47 @@ +import ai.djl.Application; +import ai.djl.Model; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelZoo; +import cn.smartjavaai.ocr.config.OcrDetModelConfig; +import cn.smartjavaai.ocr.entity.OcrBox; +import cn.smartjavaai.ocr.enums.CommonDetModelEnum; +import cn.smartjavaai.ocr.factory.OcrModelFactory; +import cn.smartjavaai.ocr.model.common.detect.OcrCommonDetModel; +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.Java2DFrameUtils; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.List; + +/** + * @author dwj + * @date 2025/4/24 + */ +@Slf4j +public class Test { + + + public static String savePath = "/Users/wenjie/Downloads/"; + //public static String image1Path = "/Users/wenjie/Documents/idea_workplace/SmartJavaAI-Demo/src/main/resources/5.jpg"; + public static String image1Path = "/Users/wenjie/Documents/idea_workplace/SmartJavaAI-Demo/src/main/resources/MJ_20250226_172200.png"; + + public static String image2Path = "/Users/wenjie/Documents/idea_workplace/SmartJavaAI-Demo/src/main/resources/MJ_20250226_172222.png"; + + public static void main(String[] args) throws IOException { +// // 加载模型 +// Model model = ModelZoo.loadModel(Criteria.builder() +// .optApplication(Application.NLP.ANY) +// .optEngine("PyTorch") +// .optModelName("Llama 3") +// .optTranslatorFactory(new Llama3TranslatorFactory()) +// .optTranslatorProvider(() -> new Llama3Translator()) +// .build()); + + + } + + +} diff --git a/smartjavaai-bom/pom.xml b/bom/pom.xml similarity index 91% rename from smartjavaai-bom/pom.xml rename to bom/pom.xml index 1357965..c50ce45 100644 --- a/smartjavaai-bom/pom.xml +++ b/bom/pom.xml @@ -6,12 +6,12 @@ cn.smartjavaai smartjavaai-parent - 1.0.22 + 1.0.27 - 1.0.22 - smartjavaai-bom - smartjavaai-bom + 1.0.27 + bom + bom 统一版本管理的 BOM 包，同时支持 import 和全量依赖 @@ -25,22 +25,27 @@ cn.smartjavaai - smartjavaai-face + face ${project.parent.version} cn.smartjavaai - smartjavaai-objectdetection + vision ${project.parent.version} cn.smartjavaai - smartjavaai-ocr + ocr ${project.parent.version} cn.smartjavaai - smartjavaai-translate + translate + ${project.parent.version} + + + cn.smartjavaai + speech ${project.parent.version} diff --git a/smartjavaai-common/pom.xml b/common/pom.xml similarity index 97% rename from smartjavaai-common/pom.xml rename to common/pom.xml index 421744a..23ad847 100644 --- a/smartjavaai-common/pom.xml +++ b/common/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.22 + 1.0.27 - smartjavaai-common - smartjavaai-common + common + common SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/config/Config.java b/common/src/main/java/cn/smartjavaai/common/config/Config.java similarity index 83% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/config/Config.java rename to common/src/main/java/cn/smartjavaai/common/config/Config.java index bf537f1..cf6ad84 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/config/Config.java +++ b/common/src/main/java/cn/smartjavaai/common/config/Config.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.io.File; +import java.nio.file.Paths; /** * 全局配置 @@ -68,19 +69,29 @@ public class Config { String osName = SystemUtil.getOsInfo().getName(); log.info("当前操作系统：{}", osName); if(osName.toLowerCase().contains("windows")){ - cachePath = SystemUtil.getUserInfo().getHomeDir() + CACHE_DIR; + cachePath = Paths.get( + SystemUtil.getUserInfo().getHomeDir(), + "smartjavaai_cache" + ).toString(); FileUtil.mkdir(cachePath); }else if(osName.toLowerCase().contains("linux")){ cachePath = "/root/" + CACHE_DIR; FileUtil.mkdir(cachePath); }else if(osName.toLowerCase().contains("mac")){ - cachePath = SystemUtil.getUserInfo().getHomeDir() + CACHE_DIR; + cachePath = Paths.get( + SystemUtil.getUserInfo().getHomeDir(), + "smartjavaai_cache" + ).toString(); FileUtil.mkdir(cachePath); }else{ - cachePath = SystemUtil.getUserInfo().getHomeDir() + CACHE_DIR; + cachePath = Paths.get( + SystemUtil.getUserInfo().getHomeDir(), + "smartjavaai_cache" + ).toString(); FileUtil.mkdir(cachePath); } } + } diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java b/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java similarity index 86% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java rename to common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java index 14a1628..2528338 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java +++ b/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java @@ -45,6 +45,12 @@ public class ModelConfig { return clazz.cast(value); } + public T getCustomParam(String key, Class clazz, T defaultValue) { + Object value = customParams.getOrDefault(key, defaultValue); + return clazz.cast(value); + } + + /** * 添加个性化配置项 */ diff --git a/common/src/main/java/cn/smartjavaai/common/cv/SmartImageFactory.java b/common/src/main/java/cn/smartjavaai/common/cv/SmartImageFactory.java new file mode 100644 index 0000000..2d0a184 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/cv/SmartImageFactory.java @@ -0,0 +1,152 @@ +package cn.smartjavaai.common.cv; + +import ai.djl.modality.cv.BufferedImageFactory; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import ai.djl.opencv.OpenCVImageFactory; +import ai.djl.util.Utils; +import cn.smartjavaai.common.utils.Base64ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import nu.pattern.OpenCV; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfByte; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * 图片处理工厂类 + * @author dwj + */ +public class SmartImageFactory { + + public enum Engine { + BUFFEREDIMAGE, + OPENCV + } + + private static volatile Engine currentEngine = Engine.BUFFEREDIMAGE; + private static volatile SmartImageFactory instance; + + public static synchronized void setEngine(Engine engine) { + if (engine == null || engine == currentEngine) { + return; + } + currentEngine = engine; + // 只在切换时注册全局 + switch (currentEngine) { + case OPENCV: + ImageFactory.setImageFactory(new OpenCVImageFactory()); + break; + case BUFFEREDIMAGE: + default: + ImageFactory.setImageFactory(new BufferedImageFactory()); + } + } + + public static synchronized SmartImageFactory getInstance() { + if (instance == null) { + instance = new SmartImageFactory(); + // 初始化全局 Engine + switch (currentEngine) { + case OPENCV: + ImageFactory.setImageFactory(new OpenCVImageFactory()); + break; + case BUFFEREDIMAGE: + default: + ImageFactory.setImageFactory(new BufferedImageFactory()); + } + } + return instance; + } + + + public Image fromBufferedImage(BufferedImage sourceImage){ + if (sourceImage == null) { + throw new IllegalArgumentException("BufferedImage 不能为空"); + } + Image image = null; + switch (currentEngine) { + case BUFFEREDIMAGE: + image = ImageFactory.getInstance().fromImage(sourceImage); + break; + case OPENCV: + // 先转 Mat + Mat mat = OpenCVUtils.image2Mat(sourceImage); + image = ImageFactory.getInstance().fromImage(mat); + break; + default: + throw new IllegalStateException("未知 Engine: " + currentEngine); + } + return image; + } + + public Image fromMat(Mat mat){ + if (mat == null) { + throw new IllegalArgumentException("mat 不能为空"); + } + Image image = null; + switch (currentEngine) { + case OPENCV: + image = ImageFactory.getInstance().fromImage(mat); + break; + case BUFFEREDIMAGE: + // 先转 Mat + BufferedImage sourceImage = OpenCVUtils.mat2Image(mat); + image = ImageFactory.getInstance().fromImage(sourceImage); + break; + default: + throw new IllegalStateException("未知 Engine: " + currentEngine); + } + return image; + } + + public Image fromBase64(String base64Image) throws IOException { + return ImageFactory.getInstance().fromUrl(base64Image); + } + + public Image fromBytes(byte[] imageData) throws IOException { + return ImageFactory.getInstance().fromInputStream(new ByteArrayInputStream(imageData)); + } + + public Image fromFile(File file) throws IOException { + return ImageFactory.getInstance().fromFile(file.toPath()); + } + + public Image fromFile(Path path) throws IOException { + return ImageFactory.getInstance().fromFile(path); + } + + public Image fromFile(String filePath) throws IOException { + if (filePath == null || filePath.trim().isEmpty()) { + throw new IllegalArgumentException("filePath 不能为空"); + } + return fromFile(Paths.get(filePath)); + } + + public Image fromPixels(int[] pixels, int width, int height){ + return ImageFactory.getInstance().fromPixels(pixels, width, height); + } + + public Image fromInputStream(InputStream inputStream) throws IOException { + return ImageFactory.getInstance().fromInputStream(inputStream); + } + + + +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java similarity index 86% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java rename to common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java index 4142637..247adca 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java +++ b/common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java @@ -32,6 +32,16 @@ public class DetectionInfo { */ private ObjectDetInfo objectDetInfo; + /** + * 目标分割信息 + */ + private InstanceSegInfo instanceSegInfo; + + /** + * 旋转框信息 + */ + private ObbDetInfo obbDetInfo; + public DetectionInfo() { diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java b/common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java rename to common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java b/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java similarity index 83% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java rename to common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java index 394f3ea..9aaa0f7 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java +++ b/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java @@ -1,5 +1,6 @@ package cn.smartjavaai.common.entity; +import ai.djl.modality.cv.Image; import lombok.Data; import java.util.List; @@ -7,13 +8,14 @@ import java.util.List; /** * 检测结果 * @author dwj - * @date 2025/4/12 */ @Data public class DetectionResponse { private List detectionInfoList; + private transient Image drawnImage; + public DetectionResponse() { } diff --git a/common/src/main/java/cn/smartjavaai/common/entity/InstanceSegInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/InstanceSegInfo.java new file mode 100644 index 0000000..6d42298 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/entity/InstanceSegInfo.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.common.entity; + +import lombok.Data; + +/** + * 目标分割信息 + * @author dwj + */ +@Data +public class InstanceSegInfo { + + /** + * 类别名称 + */ + private String className; + + /** + * 遮罩 + */ + private float[][] mask; + + public InstanceSegInfo() { + } + + public InstanceSegInfo(String className, float[][] mask) { + this.className = className; + this.mask = mask; + } +} diff --git a/common/src/main/java/cn/smartjavaai/common/entity/Language.java b/common/src/main/java/cn/smartjavaai/common/entity/Language.java new file mode 100644 index 0000000..8ee3deb --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/entity/Language.java @@ -0,0 +1,138 @@ +package cn.smartjavaai.common.entity; + +/** + * 语言枚举 + * @author dwj + */ +public enum Language { + + + EN("en", "English", "英语"), + ZH("zh", "Chinese", "中文"), + DE("de", "German", "德语"), + ES("es", "Spanish", "西班牙语"), + RU("ru", "Russian", "俄语"), + KO("ko", "Korean", "韩语"), + FR("fr", "French", "法语"), + JA("ja", "Japanese", "日语"), + PT("pt", "Portuguese", "葡萄牙语"), + TR("tr", "Turkish", "土耳其语"), + PL("pl", "Polish", "波兰语"), + CA("ca", "Catalan", "加泰罗尼亚语"), + NL("nl", "Dutch", "荷兰语"), + AR("ar", "Arabic", "阿拉伯语"), + SV("sv", "Swedish", "瑞典语"), + IT("it", "Italian", "意大利语"), + ID("id", "Indonesian", "印尼语"), + HI("hi", "Hindi", "印地语"), + FI("fi", "Finnish", "芬兰语"), + VI("vi", "Vietnamese", "越南语"), + HE("he", "Hebrew", "希伯来语"), + UK("uk", "Ukrainian", "乌克兰语"), + EL("el", "Greek", "希腊语"), + MS("ms", "Malay", "马来语"), + CS("cs", "Czech", "捷克语"), + RO("ro", "Romanian", "罗马尼亚语"), + DA("da", "Danish", "丹麦语"), + HU("hu", "Hungarian", "匈牙利语"), + TA("ta", "Tamil", "泰米尔语"), + NO("no", "Norwegian", "挪威语"), + TH("th", "Thai", "泰语"), + UR("ur", "Urdu", "乌尔都语"), + HR("hr", "Croatian", "克罗地亚语"), + BG("bg", "Bulgarian", "保加利亚语"), + LT("lt", "Lithuanian", "立陶宛语"), + LA("la", "Latin", "拉丁语"), + MI("mi", "Maori", "毛利语"), + ML("ml", "Malayalam", "马拉雅拉姆语"), + CY("cy", "Welsh", "威尔士语"), + SK("sk", "Slovak", "斯洛伐克语"), + TE("te", "Telugu", "泰卢固语"), + FA("fa", "Persian", "波斯语"), + LV("lv", "Latvian", "拉脱维亚语"), + BN("bn", "Bengali", "孟加拉语"), + SR("sr", "Serbian", "塞尔维亚语"), + AZ("az", "Azerbaijani", "阿塞拜疆语"), + SL("sl", "Slovenian", "斯洛文尼亚语"), + KN("kn", "Kannada", "卡纳达语"), + ET("et", "Estonian", "爱沙尼亚语"), + MK("mk", "Macedonian", "马其顿语"), + BR("br", "Breton", "布列塔尼语"), + EU("eu", "Basque", "巴斯克语"), + IS("is", "Icelandic", "冰岛语"), + HY("hy", "Armenian", "亚美尼亚语"), + NE("ne", "Nepali", "尼泊尔语"), + MN("mn", "Mongolian", "蒙古语"), + BS("bs", "Bosnian", "波斯尼亚语"), + KK("kk", "Kazakh", "哈萨克语"), + SQ("sq", "Albanian", "阿尔巴尼亚语"), + SW("sw", "Swahili", "斯瓦希里语"), + GL("gl", "Galician", "加利西亚语"), + MR("mr", "Marathi", "马拉地语"), + PA("pa", "Punjabi", "旁遮普语"), + SI("si", "Sinhala", "僧伽罗语"), + KM("km", "Khmer", "高棉语"), + SN("sn", "Shona", "修纳语"), + YO("yo", "Yoruba", "约鲁巴语"), + SO("so", "Somali", "索马里语"), + AF("af", "Afrikaans", "南非荷兰语"), + OC("oc", "Occitan", "奥克语"), + KA("ka", "Georgian", "格鲁吉亚语"), + BE("be", "Belarusian", "白俄罗斯语"), + TG("tg", "Tajik", "塔吉克语"), + SD("sd", "Sindhi", "信德语"), + GU("gu", "Gujarati", "古吉拉特语"), + AM("am", "Amharic", "阿姆哈拉语"), + YI("yi", "Yiddish", "意第绪语"), + LO("lo", "Lao", "老挝语"), + UZ("uz", "Uzbek", "乌兹别克语"), + FO("fo", "Faroese", "法罗语"), + HT("ht", "Haitian Creole", "海地克里奥尔语"), + PS("ps", "Pashto", "普什图语"), + TK("tk", "Turkmen", "土库曼语"), + NN("nn", "Nynorsk", "新挪威语"), + MT("mt", "Maltese", "马耳他语"), + SA("sa", "Sanskrit", "梵语"), + LB("lb", "Luxembourgish", "卢森堡语"), + MY("my", "Myanmar", "缅甸语"), + BO("bo", "Tibetan", "藏语"), + TL("tl", "Tagalog", "他加禄语"), + MG("mg", "Malagasy", "马尔加什语"), + AS("as", "Assamese", "阿萨姆语"), + TT("tt", "Tatar", "鞑靼语"), + HAW("haw", "Hawaiian", "夏威夷语"), + LN("ln", "Lingala", "林加拉语"), + HA("ha", "Hausa", "豪萨语"), + BA("ba", "Bashkir", "巴什基尔语"), + JW("jw", "Javanese", "爪哇语"), + SU("su", "Sundanese", "巽他语"), + YUE("yue", "Cantonese", "粤语"); + + private final String code; // Whisper语言代码 + private final String englishName; // 英文名称 + private final String chineseName; // 中文名称 + + Language(String code, String englishName, String chineseName) { + this.code = code; + this.englishName = englishName; + this.chineseName = chineseName; + } + + public String getCode() { + return code; + } + + public String getEnglishName() { + return englishName; + } + + public String getChineseName() { + return chineseName; + } + + @Override + public String toString() { + return code; + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java new file mode 100644 index 0000000..2836971 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java @@ -0,0 +1,32 @@ +package cn.smartjavaai.common.entity; + +import lombok.Data; + +import java.util.List; + +/** + * 定向边界框 检测结果 + * @author dwj + */ +@Data +public class ObbDetInfo { + + /** + * 类别名称 + */ + private String className; + + /** + * 检测框坐标 + */ + private RotatedBox rotatedBox; + + public ObbDetInfo() { + } + + + public ObbDetInfo(String className, RotatedBox rotatedBox) { + this.className = className; + this.rotatedBox = rotatedBox; + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java rename to common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Point.java b/common/src/main/java/cn/smartjavaai/common/entity/Point.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Point.java rename to common/src/main/java/cn/smartjavaai/common/entity/Point.java diff --git a/common/src/main/java/cn/smartjavaai/common/entity/PolygonLabel.java b/common/src/main/java/cn/smartjavaai/common/entity/PolygonLabel.java new file mode 100644 index 0000000..24d07a7 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/entity/PolygonLabel.java @@ -0,0 +1,28 @@ +package cn.smartjavaai.common.entity; + +import lombok.Data; + +import java.util.List; + +/** + * 多边形 + * @author dwj + */ +@Data +public class PolygonLabel { + + private List points; + private String text; + + public PolygonLabel(List points, String text) { + this.points = points; + this.text = text; + } + + public PolygonLabel() { + } + + public PolygonLabel(List points) { + this.points = points; + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java b/common/src/main/java/cn/smartjavaai/common/entity/R.java similarity index 97% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java rename to common/src/main/java/cn/smartjavaai/common/entity/R.java index 511a836..be494eb 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java +++ b/common/src/main/java/cn/smartjavaai/common/entity/R.java @@ -53,7 +53,7 @@ public class R { public enum Status { SUCCESS(0, "成功"), INVALID_IMAGE(1, "图像无效"), - FILE_NOT_FOUND(2, "图像文件不存在"), + FILE_NOT_FOUND(2, "文件不存在"), NO_FACE_DETECTED(3, "未检测到人脸"), PARAM_ERROR(4, "参数错误"), INVALID_VIDEO(5, "视频无效"), diff --git a/common/src/main/java/cn/smartjavaai/common/entity/RotatedBox.java b/common/src/main/java/cn/smartjavaai/common/entity/RotatedBox.java new file mode 100644 index 0000000..65de1c5 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/entity/RotatedBox.java @@ -0,0 +1,41 @@ +package cn.smartjavaai.common.entity; + +import lombok.Data; + +/** + * 旋转框 + * @author dwj + */ +@Data +public class RotatedBox { + + /** + * 左上角 + */ + private Point topLeft; + + /** + * 右上角 + */ + private Point topRight; + + /** + * 右下角 + */ + private Point bottomRight; + + /** + * 左下角 + */ + private Point bottomLeft; + + public RotatedBox(Point topLeft, Point topRight, Point bottomRight, Point bottomLeft) { + this.topLeft = topLeft; + this.topRight = topRight; + this.bottomRight = bottomRight; + this.bottomLeft = bottomLeft; + } + + public RotatedBox() { + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java b/common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java b/common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java b/common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java b/common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java b/common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java rename to common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ocr/TableStructure.java b/common/src/main/java/cn/smartjavaai/common/entity/ocr/TableStructure.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ocr/TableStructure.java rename to common/src/main/java/cn/smartjavaai/common/entity/ocr/TableStructure.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/DeviceEnum.java b/common/src/main/java/cn/smartjavaai/common/enums/DeviceEnum.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/DeviceEnum.java rename to common/src/main/java/cn/smartjavaai/common/enums/DeviceEnum.java diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java b/common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java similarity index 85% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java rename to common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java index 805350a..264df3d 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java +++ b/common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.face.enums; +package cn.smartjavaai.common.enums; /** * @author dwj diff --git a/common/src/main/java/cn/smartjavaai/common/enums/VideoSourceType.java b/common/src/main/java/cn/smartjavaai/common/enums/VideoSourceType.java new file mode 100644 index 0000000..522c501 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/enums/VideoSourceType.java @@ -0,0 +1,12 @@ +package cn.smartjavaai.common.enums; + +/** + * 视频源类型枚举 + * @author dwj + * @date 2025/8/27 + */ +public enum VideoSourceType { + STREAM, // RTSP 或 HTTP 流 + FILE, // 本地视频文件 + CAMERA; // 本地摄像头 +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java b/common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java rename to common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java b/common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java rename to common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java b/common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java rename to common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java b/common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java rename to common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/CommonPredictorFactory.java b/common/src/main/java/cn/smartjavaai/common/pool/CommonPredictorFactory.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/CommonPredictorFactory.java rename to common/src/main/java/cn/smartjavaai/common/pool/CommonPredictorFactory.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/ModelPredictorPoolManager.java b/common/src/main/java/cn/smartjavaai/common/pool/ModelPredictorPoolManager.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/ModelPredictorPoolManager.java rename to common/src/main/java/cn/smartjavaai/common/pool/ModelPredictorPoolManager.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/PredictorFactory.java b/common/src/main/java/cn/smartjavaai/common/pool/PredictorFactory.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/pool/PredictorFactory.java rename to common/src/main/java/cn/smartjavaai/common/pool/PredictorFactory.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java b/common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java similarity index 98% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java rename to common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java index ec0b109..e4ce4e8 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java +++ b/common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java @@ -10,7 +10,7 @@ import java.awt.image.BufferedImage; * @author dwj * @date 2025/6/27 */ -public class BufferedImagePreprocessor { +public class BufferedImagePreprocessor implements ImagePreprocessor{ private BufferedImage image; private DetectionRectangle rect; diff --git a/common/src/main/java/cn/smartjavaai/common/preprocess/DJLImagePreprocessor.java b/common/src/main/java/cn/smartjavaai/common/preprocess/DJLImagePreprocessor.java new file mode 100644 index 0000000..8cf1d30 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/preprocess/DJLImagePreprocessor.java @@ -0,0 +1,84 @@ +package cn.smartjavaai.common.preprocess; + +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionRectangle; +import org.opencv.core.Mat; + +import java.awt.*; +import java.awt.image.BufferedImage; + +/** + * @author dwj + */ +public class DJLImagePreprocessor implements ImagePreprocessor{ + + + private final ImagePreprocessor delegate; + private Image input = null; + + public DJLImagePreprocessor(Image image, DetectionRectangle rect) { + this.input = image; + if (input.getWrappedImage() instanceof BufferedImage) { + this.delegate = new BufferedImagePreprocessor((BufferedImage) input.getWrappedImage(), rect); + } else if (input.getWrappedImage() instanceof Mat) { + this.delegate = new OpenCVPreprocessor((Mat) input.getWrappedImage(), rect); + } else { + throw new IllegalArgumentException("Unsupported input type"); + } + } + + @Override + public DJLImagePreprocessor setExtendRatio(float ratio) { + delegate.setExtendRatio(ratio); + return this; + } + + @Override + public DJLImagePreprocessor setTargetSize(int size) { + delegate.setTargetSize(size); + return this; + } + + @Override + public DJLImagePreprocessor setCenterCropSize(int size) { + delegate.setCenterCropSize(size); + return this; + } + + @Override + public DJLImagePreprocessor enableSquarePadding(boolean enable) { + delegate.enableSquarePadding(enable); + return this; + } + + @Override + public DJLImagePreprocessor enableScaling(boolean enable) { + delegate.enableScaling(enable); + return this; + } + + @Override + public DJLImagePreprocessor enableCenterCrop(boolean enable) { + delegate.enableCenterCrop(enable); + return this; + } + + @Override + public ImagePreprocessor setPaddingColor(Color color) { + return delegate.setPaddingColor(color); + } + + + @Override + public Image process() { + Object result = delegate.process(); + if (result instanceof BufferedImage) { + return SmartImageFactory.getInstance().fromBufferedImage((BufferedImage) result); + } else if (result instanceof Mat) { + return SmartImageFactory.getInstance().fromMat((Mat) result); + } + throw new IllegalStateException("Unsupported process result: " + result.getClass()); + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/preprocess/ImagePreprocessor.java b/common/src/main/java/cn/smartjavaai/common/preprocess/ImagePreprocessor.java new file mode 100644 index 0000000..396e080 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/preprocess/ImagePreprocessor.java @@ -0,0 +1,28 @@ +package cn.smartjavaai.common.preprocess; + +import java.awt.*; + +/** + * 图片预处理 + * @author dwj + */ +public interface ImagePreprocessor { + + + ImagePreprocessor setExtendRatio(float ratio); + + ImagePreprocessor setTargetSize(int size); + + ImagePreprocessor setCenterCropSize(int size); + + ImagePreprocessor enableSquarePadding(boolean enable); + + ImagePreprocessor enableScaling(boolean enable); + + ImagePreprocessor enableCenterCrop(boolean enable); + + ImagePreprocessor setPaddingColor(Color color); + + T process(); + +} diff --git a/common/src/main/java/cn/smartjavaai/common/preprocess/OpenCVPreprocessor.java b/common/src/main/java/cn/smartjavaai/common/preprocess/OpenCVPreprocessor.java new file mode 100644 index 0000000..44dddf9 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/preprocess/OpenCVPreprocessor.java @@ -0,0 +1,154 @@ +package cn.smartjavaai.common.preprocess; + +import cn.smartjavaai.common.entity.DetectionRectangle; +import org.opencv.core.Mat; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; + +/** + * @author dwj + */ +public class OpenCVPreprocessor implements ImagePreprocessor { + + private Mat image; + private DetectionRectangle rect; + private float extendRatio = 1; + private int targetSize = 128; + private int centerCropSize = 80; + + private Scalar paddingColor = new Scalar(127, 127, 127); // 默认灰色 + private boolean enableSquarePadding = true; + private boolean enableScaling = true; + private boolean enableCenterCrop = false; + + public OpenCVPreprocessor(Mat image, DetectionRectangle rect) { + this.image = image; + this.rect = rect; + } + + @Override + public OpenCVPreprocessor setExtendRatio(float ratio) { + this.extendRatio = ratio; + return this; + } + + @Override + public OpenCVPreprocessor setTargetSize(int size) { + this.targetSize = size; + return this; + } + + @Override + public OpenCVPreprocessor setCenterCropSize(int size) { + this.centerCropSize = size; + return this; + } + + @Override + public OpenCVPreprocessor enableSquarePadding(boolean enable) { + this.enableSquarePadding = enable; + return this; + } + + @Override + public OpenCVPreprocessor enableScaling(boolean enable) { + this.enableScaling = enable; + return this; + } + + @Override + public OpenCVPreprocessor enableCenterCrop(boolean enable) { + this.enableCenterCrop = enable; + return this; + } + + @Override + public OpenCVPreprocessor setPaddingColor(java.awt.Color color) { + this.paddingColor = new Scalar(color.getBlue(), color.getGreen(), color.getRed()); + return this; + } + + @Override + public Mat process() { + // Step 1: 裁剪 + 扩展 + Mat cropped = cropAndExtend(); + + // Step 2: 填充正方形 + Mat squared = enableSquarePadding ? squarePadding(cropped) : cropped; + + // Step 3: 缩放 + Mat scaled = enableScaling ? scaleToTarget(squared) : squared; + + // Step 4: CenterCrop + Mat finalResult = enableCenterCrop ? centerCrop(scaled) : scaled; + + return finalResult; + } + + /** + * 检测框扩展及裁剪 + */ + private Mat cropAndExtend() { + int x = rect.x; + int y = rect.y; + int width = rect.width; + int height = rect.height; + + int extendX = Math.round(width * extendRatio); + int extendY = Math.round(height * extendRatio); + + int left = Math.max(0, x - extendX); + int right = Math.min(image.width(), x + width + extendX); + int top = Math.max(0, y - extendY); + int bottom = Math.min(image.height(), y + height + extendY); + + int origRoiWidth = right - left; + int origRoiHeight = bottom - top; + int longSide = Math.max(origRoiWidth, origRoiHeight); + + // 计算可扩展空间（不超出原图边界） + int extendLeft = Math.min(left, (longSide - origRoiWidth) / 2); + int extendRight = Math.min(image.width() - right, (longSide - origRoiWidth + 1) / 2); + int extendTop = Math.min(top, (longSide - origRoiHeight) / 2); + int extendBottom = Math.min(image.height() - bottom, (longSide - origRoiHeight + 1) / 2); + + int expandedLeft = left - extendLeft; + int expandedRight = right + extendRight; + int expandedTop = top - extendTop; + int expandedBottom = bottom + extendBottom; + + Rect roi = new Rect(expandedLeft, expandedTop, expandedRight - expandedLeft, expandedBottom - expandedTop); + return new Mat(image, roi).clone(); // clone 避免与原图共享内存 + } + + /** + * 填充为正方形 + */ + private Mat squarePadding(Mat src) { + int longSide = Math.max(src.width(), src.height()); + Mat squared = new Mat(new Size(longSide, longSide), src.type(), paddingColor); + int xOffset = (longSide - src.width()) / 2; + int yOffset = (longSide - src.height()) / 2; + src.copyTo(squared.submat(yOffset, yOffset + src.height(), xOffset, xOffset + src.width())); + return squared; + } + + /** + * 缩放到目标大小 + */ + private Mat scaleToTarget(Mat src) { + Mat result = new Mat(); + Imgproc.resize(src, result, new Size(targetSize, targetSize), 0, 0, Imgproc.INTER_AREA); + return result; + } + + /** + * CenterCrop + */ + private Mat centerCrop(Mat src) { + int startX = (src.width() - centerCropSize) / 2; + int startY = (src.height() - centerCropSize) / 2; + Rect roi = new Rect(startX, startY, centerCropSize, centerCropSize); + return new Mat(src, roi).clone(); + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java diff --git a/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java new file mode 100644 index 0000000..9b590bc --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java @@ -0,0 +1,608 @@ +package cn.smartjavaai.common.utils; + +import ai.djl.ndarray.NDArray; +import ai.djl.util.RandomUtils; +import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceSearchResult; +import cn.smartjavaai.common.entity.face.HeadPose; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.commons.lang3.StringUtils; +import org.opencv.core.Mat; +import org.opencv.core.Scalar; +import org.opencv.imgproc.Imgproc; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.awt.image.ComponentSampleModel; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * @author dwj + */ +public class BufferedImageUtils { + + /** + * 拷贝图片 + * @param src + * @return + */ + public static BufferedImage copyBufferedImage(BufferedImage src) { + BufferedImage copy = new BufferedImage(src.getWidth(), src.getHeight(), src.getType()); + Graphics2D g = copy.createGraphics(); + g.drawImage(src, 0, 0, null); + g.dispose(); + return copy; + } + + /** + * 对图像解码返回BGR格式矩阵数据 + * + * @param image + * @return + */ + public static byte[] getMatrixBGR(BufferedImage image) { + byte[] matrixBGR; + if (isBGR3Byte(image)) { + matrixBGR = (byte[]) image.getData().getDataElements(0, 0, image.getWidth(), image.getHeight(), null); + } else { + // ARGB格式图像数据 + int intrgb[] = image.getRGB(0, 0, image.getWidth(), image.getHeight(), null, 0, image.getWidth()); + matrixBGR = new byte[image.getWidth() * image.getHeight() * 3]; + // ARGB转BGR格式 + for (int i = 0, j = 0; i < intrgb.length; ++i, j += 3) { + matrixBGR[j] = (byte) (intrgb[i] & 0xff); + matrixBGR[j + 1] = (byte) ((intrgb[i] >> 8) & 0xff); + matrixBGR[j + 2] = (byte) ((intrgb[i] >> 16) & 0xff); + } + } + return matrixBGR; + } + + /** + * 推断图像是否为BGR格式 + * + * @return + */ + public static boolean isBGR3Byte(BufferedImage image) { + return equalBandOffsetWith3Byte(image, new int[]{0, 1, 2}); + } + + /** + * @param image + * @param bandOffset 用于推断通道顺序 + * @return + */ + private static boolean equalBandOffsetWith3Byte(BufferedImage image, int[] bandOffset) { + if (image.getType() == BufferedImage.TYPE_3BYTE_BGR) { + if (image.getData().getSampleModel() instanceof ComponentSampleModel) { + ComponentSampleModel sampleModel = (ComponentSampleModel) image.getData().getSampleModel(); + if (Arrays.equals(sampleModel.getBandOffsets(), bandOffset)) { + return true; + } + } + } + return false; + } + + public static BufferedImage bgrToBufferedImage(byte[] data, int width, int height) { + int type = BufferedImage.TYPE_3BYTE_BGR; + // bgr to rgb + byte b; + for (int i = 0; i < data.length; i = i + 3) { + b = data[i]; + data[i] = data[i + 2]; + data[i + 2] = b; + } + BufferedImage image = new BufferedImage(width, height, type); + image.getRaster().setDataElements(0, 0, width, height, data); + return image; + } + + /** + * 检查图像是否有效 + * @param image + * @return + */ + public static boolean isImageValid(BufferedImage image) { + // 检查是否为 null 或尺寸异常（如宽高为0） + return image != null && image.getWidth() > 0 && image.getHeight() > 0; + } + + /** + * 画检测框 + * + * @param image + * @param x + * @param y + * @param width + * @param height + */ + public static void drawRect(BufferedImage image, int x, int y, int width, int height) { + // 将绘制图像转换为Graphics2D + Graphics2D g = (Graphics2D) image.getGraphics(); + try { + g.setColor(new Color(0, 255, 0)); + // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角 + BasicStroke bStroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER); + g.setStroke(bStroke); + g.drawRect(x, y, width, height); + } finally { + g.dispose(); + } + } + + /** + * 保存BufferedImage图片 + * @param image + * @param outputPath + * @param formatName + * @throws IOException + */ + public static void saveBufferedImage(BufferedImage image, String outputPath, String formatName) throws IOException { + if (image == null) { + throw new IllegalArgumentException("BufferedImage 不能为空"); + } + if (outputPath == null || outputPath.isEmpty()) { + throw new IllegalArgumentException("输出路径不能为空"); + } + if (formatName == null || formatName.isEmpty()) { + throw new IllegalArgumentException("格式不能为空"); + } + + Path path = Paths.get(outputPath); + Path parent = path.getParent(); + if (parent != null && !Files.exists(parent)) { + Files.createDirectories(parent); // 自动创建父目录 + } + File outFile = path.toFile(); + boolean result = ImageIO.write(image, formatName, outFile); + if (!result) { + throw new IOException("保存图片失败，不支持的格式: " + formatName); + } + } + + + /** + * 默认保存图片格式为png + * @param image + * @param outputPath + * @throws IOException + */ + public static void saveImage(BufferedImage image, String outputPath) throws IOException { + saveBufferedImage(image, outputPath, "png"); + } + + + + /** + * 画检测框(有倾斜角) + * + * @param image + * @param box + */ + public static void drawRect(BufferedImage image, NDArray box) { + float[] points = box.toFloatArray(); + int[] xPoints = new int[5]; + int[] yPoints = new int[5]; + + for (int i = 0; i < 4; i++) { + xPoints[i] = (int) points[2 * i]; + yPoints[i] = (int) points[2 * i + 1]; + } + xPoints[4] = xPoints[0]; + yPoints[4] = yPoints[0]; + + // 将绘制图像转换为Graphics2D + Graphics2D g = (Graphics2D) image.getGraphics(); + try { + g.setColor(new Color(0, 255, 0)); + // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角 + BasicStroke bStroke = new BasicStroke(4, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER); + g.setStroke(bStroke); + g.drawPolyline(xPoints, yPoints, 5); // xPoints, yPoints, nPoints + } finally { + g.dispose(); + } + } + + /** + * 画检测框(有倾斜角)和文本 + * + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(BufferedImage image, NDArray box, String text) { + float[] points = box.toFloatArray(); + int[] xPoints = new int[5]; + int[] yPoints = new int[5]; + + for (int i = 0; i < 4; i++) { + xPoints[i] = (int) points[2 * i]; + yPoints[i] = (int) points[2 * i + 1]; + } + xPoints[4] = xPoints[0]; + yPoints[4] = yPoints[0]; + + // 将绘制图像转换为Graphics2D + Graphics2D g = (Graphics2D) image.getGraphics(); + try { + int fontSize = 32; + Font font = new Font("楷体", Font.PLAIN, fontSize); + g.setFont(font); + g.setColor(new Color(0, 0, 255)); + // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角 + BasicStroke bStroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER); + g.setStroke(bStroke); + g.drawPolyline(xPoints, yPoints, 5); // xPoints, yPoints, nPoints + g.drawString(text, xPoints[0], yPoints[0]); + } finally { + g.dispose(); + } + } + + + /** + * 显示文字 + * + * @param image + * @param text + * @param x + * @param y + */ + public static void drawImageText(BufferedImage image, String text, int x, int y) { + Graphics graphics = image.getGraphics(); + int fontSize = 32; + Font font = new Font("楷体", Font.PLAIN, fontSize); + try { + graphics.setFont(font); + graphics.setColor(new Color(0, 0, 255)); + int strWidth = graphics.getFontMetrics().stringWidth(text); + graphics.drawString(text, x, y); + } finally { + graphics.dispose(); + } + } + + + /** + * 画检测框(有倾斜角)和文本 + * + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(BufferedImage image, DetectionRectangle box, String text, Color color) { + // 将绘制图像转换为Graphics2D + Graphics2D graphics = (Graphics2D) image.getGraphics(); + try { + drawRectAndText(graphics, box, text, color); + } finally { + graphics.dispose(); + } + } + + + /** + * 画检测框(有倾斜角)和文本 + * + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(BufferedImage image, DetectionRectangle box, String text, int fontSize) { + Color color = new Color(255, 0, 0); + // 将绘制图像转换为Graphics2D + Graphics2D graphics = (Graphics2D) image.getGraphics(); + graphics.setFont(new Font("楷体", Font.PLAIN, fontSize)); + try { + drawRectAndText(graphics, box, text, color); + } finally { + graphics.dispose(); + } + } + + /** + * 画检测框(有倾斜角)和文本 + * + * @param graphics + * @param box + * @param text + */ + public static void drawRectAndText(Graphics2D graphics, DetectionRectangle box, String text, Color color) { + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + graphics.setColor(color);// 边框颜色 + graphics.drawRect(box.getX(), box.getY(), box.getWidth(), box.getHeight()); + Graphics2DUtils.drawText(graphics, text, box.getX(), box.getY(), stroke, 4); + } + + /** + * 绘制检测框 + * @param sourceImage + * @param detectionResponse + * @throws IOException + */ + public static void drawFaceSearchResult(BufferedImage sourceImage, DetectionResponse detectionResponse, String displayField) { + if(!BufferedImageUtils.isImageValid(sourceImage)){ + throw new IllegalArgumentException("图像无效"); + } + if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){ + throw new IllegalArgumentException("无目标数据"); + } + Graphics2D graphics = sourceImage.createGraphics(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){ + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight()); + //绘制人脸关键点 + if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null && + !detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){ + Graphics2DUtils.drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints()); + //人脸查询结果 + if(detectionInfo.getFaceInfo().getFaceSearchResults() != null){ + for (FaceSearchResult faceSearchResult : detectionInfo.getFaceInfo().getFaceSearchResults()){ + if(StringUtils.isNotBlank(faceSearchResult.getMetadata())){ + JsonObject metadata = GsonUtils.parseToJsonObject(faceSearchResult.getMetadata()); + JsonElement nameElement = metadata.get("name"); + if(metadata.has("name")){ + Graphics2DUtils.drawText(graphics, nameElement.getAsString(), rectangle.getX(), rectangle.getY(), stroke, 4); + } + } + } + } + } + } + graphics.dispose(); + } + + + + + + /** + * 绘制检测框 + * @param sourceImage + * @param detectionResponse + * @throws IOException + */ + public static void drawBoundingBoxes(BufferedImage sourceImage, DetectionResponse detectionResponse) { + if(!BufferedImageUtils.isImageValid(sourceImage)){ + throw new IllegalArgumentException("图像无效"); + } + if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){ + throw new IllegalArgumentException("无目标数据"); + } + Graphics2D graphics = sourceImage.createGraphics(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){ + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight()); + //绘制人脸关键点 + if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null && + !detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){ + Graphics2DUtils.drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints()); +// Graphics2DUtils.drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4); + //人脸属性 + if(detectionInfo.getFaceInfo().getFaceAttribute() != null){ + FaceAttribute faceAttribute = detectionInfo.getFaceInfo().getFaceAttribute(); + drawFaceAttribute(faceAttribute, rectangle, graphics); + } + } + //绘制目标检测信息 + if(detectionInfo.getObjectDetInfo() != null){ + String className = detectionInfo.getObjectDetInfo().getClassName(); + Graphics2DUtils.drawText(graphics, className, rectangle.getX(), rectangle.getY(), stroke, 4); + } + } + graphics.dispose(); + } + + /** + * 绘制矩形框和文字 + * + * @param sourceImage + * @param detectionInfo + */ + public static void drawRectAndText(BufferedImage sourceImage, DetectionInfo detectionInfo) { + if(!BufferedImageUtils.isImageValid(sourceImage)){ + throw new IllegalArgumentException("图像无效"); + } + if(Objects.isNull(detectionInfo)){ + throw new IllegalArgumentException("无目标数据"); + } + Graphics2D graphics = sourceImage.createGraphics(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight()); + //绘制人脸关键点 + if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null && + !detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){ + Graphics2DUtils.drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints()); +// Graphics2DUtils.drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4); + //人脸属性 + if(detectionInfo.getFaceInfo().getFaceAttribute() != null){ + FaceAttribute faceAttribute = detectionInfo.getFaceInfo().getFaceAttribute(); + drawFaceAttribute(faceAttribute, rectangle, graphics); + } + } + //绘制目标检测信息 + if(detectionInfo.getObjectDetInfo() != null){ + String className = detectionInfo.getObjectDetInfo().getClassName(); + Graphics2DUtils.drawText(graphics, className, rectangle.getX(), rectangle.getY(), stroke, 4); + } + graphics.dispose(); + } + + + /** + * 绘制人脸属性 + * @param faceAttribute + * @param rectangle + * @param graphics + */ + public static void drawFaceAttribute(FaceAttribute faceAttribute, DetectionRectangle rectangle, Graphics2D graphics){ + List lines = new ArrayList<>(); + if (faceAttribute.getGenderType() != null) { + lines.add("性别: " + faceAttribute.getGenderType().name()); + } + if (faceAttribute.getAge() != null) { + lines.add("年龄: " + faceAttribute.getAge()); + } + if (faceAttribute.getWearingMask() != null) { + lines.add("口罩: " + (faceAttribute.getWearingMask() ? "是" : "否")); + } + if (faceAttribute.getLeftEyeStatus() != null && faceAttribute.getRightEyeStatus() != null) { + lines.add("眼睛: " + faceAttribute.getLeftEyeStatus().name() + "/" + faceAttribute.getRightEyeStatus().name()); + } + if (faceAttribute.getHeadPose() != null) { + HeadPose pose = faceAttribute.getHeadPose(); + String pitch = pose.getPitch() != null ? String.valueOf(pose.getPitch().intValue()) : "-"; + String yaw = pose.getYaw() != null ? String.valueOf(pose.getYaw().intValue()) : "-"; + String roll = pose.getRoll() != null ? String.valueOf(pose.getRoll().intValue()) : "-"; + lines.add("姿态: P=" + pitch + " Y=" + yaw + " R=" + roll); + } + if (!lines.isEmpty()) { + Graphics2DUtils.drawMultilineTextWithBackground(graphics, lines, rectangle.getX(), rectangle.getY()); // 适当偏移 + } + } + + + + public static void drawPolygonWithText(BufferedImage image,List polygonLabelList, int fontSize) { + Font font = new Font("楷体", Font.PLAIN, fontSize); + Stroke stroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER); + Color color = new Color(0, 0, 255); + Graphics2D g = (Graphics2D) image.getGraphics(); + for (PolygonLabel polygonLabel : polygonLabelList){ + drawPolygonWithText(g, polygonLabel.getPoints(), polygonLabel.getText(), font, color, stroke); + } + } + + /** + * 绘制多边形及文字 + * @param g Graphics2D + * @param points 多边形顶点 + * @param text 绘制的文字（可为空） + * @param font 字体 + * @param color 颜色 + * @param stroke 画笔样式 + */ + public static void drawPolygonWithText(Graphics2D g, List points, + String text, int fontSize) { + Font font = new Font("楷体", Font.PLAIN, fontSize); + Stroke stroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER); + Color color = new Color(0, 0, 255); + drawPolygonWithText(g, points, text, font, color, stroke); + } + + + /** + * 绘制多边形及文字 + * @param g Graphics2D + * @param points 多边形顶点 + * @param text 绘制的文字（可为空） + * @param font 字体 + * @param color 颜色 + * @param stroke 画笔样式 + */ + public static void drawPolygonWithText(Graphics2D g, List points, + String text, Font font, + Color color, Stroke stroke) { + if (points == null || points.size() < 3) { + return; + } + + int[] xPoints = points.stream().mapToInt(p -> (int) p.getX()).toArray(); + int[] yPoints = points.stream().mapToInt(p -> (int) p.getY()).toArray(); + + g.setFont(font); + g.setColor(color); + g.setStroke(stroke); + // 绘制多边形 + g.drawPolygon(xPoints, yPoints, points.size()); + // 绘制文字（默认放在第一个点） + if (text != null && !text.isEmpty()) { + g.drawString(text, xPoints[0], yPoints[0]); + } + } + + /** + * 绘制关键点 + * @param graphics + * @param points + * @param color + */ + public static void drawKeyPoints(Graphics2D graphics, List points, Color color){ + if(points == null || points.isEmpty()){ + return; + } + for (Point point : points){ + //绘制关键点 + graphics.setColor(color); + graphics.drawRect((int)point.getX(), (int)point.getY(), 2, 2); + } + } + + + /** + * 绘制检测框 + * @param sourceImage + * @param detectionResponse + * @throws IOException + */ + public static void drawFaceSearchResult(Graphics2D graphics, DetectionInfo detectionInfo, String text) { + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight()); + //绘制人脸关键点 + //人脸查询结果 + if(detectionInfo.getFaceInfo().getFaceSearchResults() != null){ + for (FaceSearchResult faceSearchResult : detectionInfo.getFaceInfo().getFaceSearchResults()){ + if(StringUtils.isNotBlank(faceSearchResult.getMetadata())){ + JsonObject metadata = GsonUtils.parseToJsonObject(faceSearchResult.getMetadata()); + JsonElement nameElement = metadata.get("name"); + if(metadata.has("name")){ + Graphics2DUtils.drawText(graphics, nameElement.getAsString(), rectangle.getX(), rectangle.getY(), stroke, 4); + } + } + } + } + graphics.dispose(); + } + +} diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/opencv/OcrNDArrayUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java similarity index 51% rename from smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/opencv/OcrNDArrayUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java index 0673f9d..1bd73e4 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/opencv/OcrNDArrayUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java @@ -1,37 +1,66 @@ -package cn.smartjavaai.ocr.opencv; +package cn.smartjavaai.common.utils; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Point; import ai.djl.ndarray.NDArray; +import cn.smartjavaai.common.entity.R; import org.opencv.core.CvType; import org.opencv.core.Mat; -import org.opencv.core.MatOfPoint; -import org.opencv.core.Point; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Objects; + /** - * NDArray Utils - * + * @author dwj */ -public class OcrNDArrayUtils { +public class DJLCommonUtils { + + private static final List SUPPORTED_PROTOCOLS = Arrays.asList( + "file://", + "http://", + "https://", + "jar://", + "djl://", + "s3://", + "hdfs://" + ); + /** - * Mat To MatOfPoint - * @param mat + * 检查模型目录中是否存在 "serving.properties" 文件 + * + * @param modelPath 模型目录路径 + * @return true 表示存在，false 表示不存在 + */ + public static boolean isServingPropertiesExists(Path modelPath) { + if (modelPath == null || !Files.exists(modelPath)) { + return false; + } + // 确定目录路径 + Path dirPath = Files.isDirectory(modelPath) ? modelPath : modelPath.getParent(); + if (dirPath == null) { + return false; // 可能是根目录的文件 + } + + // 判断目录下的 serving.properties 是否存在 + Path servingFile = dirPath.resolve("serving.properties"); + return Files.exists(servingFile); + } + + /** + * 判断 NDArray 是否为空 + * @param ndArray * @return */ - public static MatOfPoint matToMatOfPoint(Mat mat) { - int rows = mat.rows(); - MatOfPoint matOfPoint = new MatOfPoint(); - - List list = new ArrayList<>(); - for (int i = 0; i < rows; i++) { - Point point = new Point((float) mat.get(i, 0)[0], (float) mat.get(i, 1)[0]); - list.add(point); - } - matOfPoint.fromList(list); - - return matOfPoint; + public static boolean isNDArrayEmpty(NDArray ndArray){ + return Objects.isNull(ndArray) || ndArray.size() == 0; } + /** * float NDArray To float[][] Array * @param ndArray @@ -51,65 +80,6 @@ public class OcrNDArrayUtils { return arr; } - /** - * Mat To double[][] Array - * @param mat - * @return - */ - public static double[][] matToDoubleArray(Mat mat) { - int rows = mat.rows(); - int cols = mat.cols(); - - double[][] doubles = new double[rows][cols]; - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - doubles[i][j] = mat.get(i, j)[0]; - } - } - - return doubles; - } - - /** - * Mat To float[][] Array - * @param mat - * @return - */ - public static float[][] matToFloatArray(Mat mat) { - int rows = mat.rows(); - int cols = mat.cols(); - - float[][] floats = new float[rows][cols]; - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - floats[i][j] = (float) mat.get(i, j)[0]; - } - } - - return floats; - } - - /** - * Mat To byte[][] Array - * @param mat - * @return - */ - public static byte[][] matToUint8Array(Mat mat) { - int rows = mat.rows(); - int cols = mat.cols(); - - byte[][] bytes = new byte[rows][cols]; - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - bytes[i][j] = (byte) mat.get(i, j)[0]; - } - } - - return bytes; - } /** * float NDArray To float[][] Array @@ -172,57 +142,44 @@ public class OcrNDArrayUtils { return mat; } - /** - * float[][] Array To Mat - * @param arr - * @return - */ - public static Mat floatArrayToMat(float[][] arr) { - int rows = arr.length; - int cols = arr[0].length; - Mat mat = new Mat(rows, cols, CvType.CV_32F); - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - mat.put(i, j, arr[i][j]); - } - } - - return mat; - } - - /** - * byte[][] Array To Mat - * @param arr - * @return - */ - public static Mat uint8ArrayToMat(byte[][] arr) { - int rows = arr.length; - int cols = arr[0].length; - Mat mat = new Mat(rows, cols, CvType.CV_8U); - - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - mat.put(i, j, arr[i][j]); - } - } - - return mat; - } /** * List To Mat * @param points * @return */ - public static Mat toMat(List points) { + public static Mat toMat(List points) { Mat mat = new Mat(points.size(), 2, CvType.CV_32F); for (int i = 0; i < points.size(); i++) { ai.djl.modality.cv.output.Point point = points.get(i); mat.put(i, 0, (float) point.getX()); mat.put(i, 1, (float) point.getY()); } - return mat; } + + /** + * 构建一个空的 DetectedObjects 对象 + * @return + */ + public static DetectedObjects buildEmptyDetectedObjects(){ + List classNames = new ArrayList<>(); + List probabilities = new ArrayList<>(); + List boxes = new ArrayList<>(); + return new DetectedObjects(classNames, probabilities, boxes); + } + + /** + * 判断路径是否以已知协议开头 + * @param path 模型路径 + * @return 是否以支持的协议开头 + */ + public static boolean hasSupportedProtocol(String path) { + if (path == null || path.isEmpty()) { + return false; + } + return SUPPORTED_PROTOCOLS.stream().anyMatch(path::startsWith); + } + + } diff --git a/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java new file mode 100644 index 0000000..218bffc --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java @@ -0,0 +1,162 @@ +package cn.smartjavaai.common.utils; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 文件操作工具类 + * @author dwj + * @date 2025/4/4 + */ +public class FileUtils { + + /** + * 检查文件是否存在 + * @param filePath + * @return + */ + public static boolean isFileExists(String filePath) { + File file = new File(filePath); + return file.exists() && !file.isDirectory(); // 确保是文件且存在 + } + + /** + * 检查目录是否存在 + * @param path + * @return + */ + public static boolean isValidDirectory(String path) { + File file = new File(path); + return file.exists() && file.isDirectory(); + } + + /** + * 查找指定目录下指定后缀的文件 + * + * @param dir 目录 + * @param suffix 文件后缀，例如 ".txt"、".wav" + * @param recursive 是否递归子目录 + * @return 文件列表 + */ + public static List findFilesWithSuffix(File dir, String suffix, boolean recursive) { + List result = new ArrayList<>(); + if (dir == null || !dir.exists() || !dir.isDirectory()) { + return result; + } + searchFiles(dir, suffix, recursive, result); + return result; + } + + // 搜索方法 + private static void searchFiles(File dir, String suffix, boolean recursive, List result) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + if (file.isDirectory()) { + if(file.getName().endsWith(suffix)){ + result.add(file); + continue; + } + if (recursive) { + searchFiles(file, suffix, true, result); + } + } else if (file.isFile() && file.getName().endsWith(suffix)) { + result.add(file); + } + } + } + + /** + * 查找指定目录下指定文件名的文件 + * + * @param dir 目录 + * @param fileName 文件名（精确匹配） + * @param recursive 是否递归子目录 + * @return 文件列表 + */ + public static List findFilesByName(File dir, String fileName, boolean recursive) { + List result = new ArrayList<>(); + if (dir == null || !dir.exists() || !dir.isDirectory() || fileName == null) { + return result; + } + searchByName(dir, fileName, recursive, result); + return result; + } + + // 递归搜索方法 + private static void searchByName(File dir, String fileName, boolean recursive, List result) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + if (file.isDirectory()) { + if (file.getName().equals(fileName)){ + result.add(file); + continue; + } + if (recursive) { + searchByName(file, fileName, true, result); + } + } else if (file.isFile() && file.getName().equals(fileName)) { + result.add(file); + } + } + } + + /** + * 将文件列表转换为绝对路径字符串 + * + * @param files 文件列表 + * @return 绝对路径字符串，用逗号分隔 + */ + public static String joinAbsolutePaths(List files) { + if (files == null || files.isEmpty()) { + return ""; + } + return files.stream() + .map(File::getAbsolutePath) + .collect(Collectors.joining(",")); + } + + /** + * 在指定目录中查找文件名包含指定关键字的文件，可选指定后缀。 + * + * @param dirPath 要搜索的目录路径 + * @param keyword 文件名包含的关键字（可为 null） + * @param extension 文件后缀名（例如 ".wav"，可为 null） + * @param recursive 是否递归搜索子目录 + * @return 匹配的文件列表 + */ + public static List searchFiles(String dirPath, String keyword, String extension, boolean recursive) { + List result = new ArrayList<>(); + File dir = new File(dirPath); + + if (!dir.exists() || !dir.isDirectory()) { + System.err.println("目录不存在或不是目录：" + dirPath); + return result; + } + + File[] files = dir.listFiles(); + if (files == null) return result; + + for (File file : files) { + if (file.isDirectory() && recursive) { + // 递归子目录 + result.addAll(searchFiles(file.getAbsolutePath(), keyword, extension, true)); + } else if (file.isFile()) { + String name = file.getName().toLowerCase(); + boolean matchKeyword = (keyword == null || name.contains(keyword.toLowerCase())); + boolean matchExt = (extension == null || name.endsWith(extension.toLowerCase())); + + if (matchKeyword && matchExt) { + result.add(file); + } + } + } + return result; + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/utils/Graphics2DUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/Graphics2DUtils.java new file mode 100644 index 0000000..4c50a1f --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/Graphics2DUtils.java @@ -0,0 +1,74 @@ +package cn.smartjavaai.common.utils; + +import cn.smartjavaai.common.entity.Point; + +import java.awt.*; +import java.util.List; + +/** + * @author dwj + */ +public class Graphics2DUtils { + + + /** + * 绘制文本 + * @param g + * @param text + * @param x + * @param y + * @param stroke + * @param padding + */ + public static void drawText(Graphics2D g, String text, int x, int y, int stroke, int padding) { + FontMetrics metrics = g.getFontMetrics(); + x += stroke / 2; + y += stroke / 2; + int width = metrics.stringWidth(text) + padding * 2 - stroke / 2; + int height = metrics.getHeight() + metrics.getDescent(); + int ascent = metrics.getAscent(); + y = Math.max(0, y - height); + java.awt.Rectangle background = new java.awt.Rectangle(x, y, width, height); + g.fill(background); + g.setPaint(Color.WHITE); + g.drawString(text, x + padding, y + ascent); + } + + /** + * 绘制人脸关键点 + * @param g + * @param keyPoints + */ + public static void drawLandmarks(Graphics2D g, List keyPoints) { + g.setColor(new Color(246, 96, 0)); + BasicStroke bStroke = new BasicStroke(4.0F, 0, 0); + g.setStroke(bStroke); + for (Point point : keyPoints){ + g.drawRect((int)point.getX(), (int)point.getY(), 2, 2); + } + } + + + public static void drawMultilineTextWithBackground(Graphics2D g, List lines, int x, int y) { + Font font = new Font("SansSerif", Font.PLAIN, 14); + g.setFont(font); + FontMetrics fm = g.getFontMetrics(); + int lineHeight = fm.getHeight(); + int maxWidth = lines.stream().mapToInt(fm::stringWidth).max().orElse(0); + + int padding = 4; + int boxWidth = maxWidth + padding * 2; + int boxHeight = lineHeight * lines.size() + padding * 2; + + // 背景矩形 + g.setColor(new Color(0, 0, 0, 128)); + g.fillRoundRect(x, y, boxWidth, boxHeight, 8, 8); + + // 绘制每一行文字 + g.setColor(Color.WHITE); + for (int i = 0; i < lines.size(); i++) { + g.drawString(lines.get(i), x + padding, y + padding + (i + 1) * lineHeight - 4); + } + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/utils/GsonUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/GsonUtils.java new file mode 100644 index 0000000..3fa22ca --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/GsonUtils.java @@ -0,0 +1,69 @@ +package cn.smartjavaai.common.utils; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; + +/** + * @author dwj + */ +public class GsonUtils { + + private static final Gson GSON = new Gson(); + + private GsonUtils() { + // 私有构造，防止实例化 + } + + /** + * 将 JSON 字符串安全转换为 JsonObject + * + * @param jsonStr JSON 字符串 + * @return JsonObject，如果解析失败返回 null + */ + public static JsonObject parseToJsonObject(String jsonStr) { + if (jsonStr == null || jsonStr.isEmpty()) { + return null; + } + try { + return JsonParser.parseString(jsonStr).getAsJsonObject(); + } catch (JsonSyntaxException | IllegalStateException e) { + // 解析失败返回 null + return null; + } + } + + /** + * 将 JSON 字符串转换为指定类型对象 + * + * @param jsonStr JSON 字符串 + * @param clazz 目标类型 + * @param 类型参数 + * @return 对象实例，如果解析失败返回 null + */ + public static T fromJson(String jsonStr, Class clazz) { + if (jsonStr == null || jsonStr.isEmpty()) { + return null; + } + try { + return GSON.fromJson(jsonStr, clazz); + } catch (JsonSyntaxException e) { + return null; + } + } + + /** + * 将对象转换为 JSON 字符串 + * + * @param obj 对象 + * @return JSON 字符串，如果对象为 null 返回 null + */ + public static String toJson(Object obj) { + if (obj == null) { + return null; + } + return GSON.toJson(obj); + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java new file mode 100644 index 0000000..2eb206a --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java @@ -0,0 +1,498 @@ +package cn.smartjavaai.common.utils; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.CategoryMask; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.opencv.OpenCVImageFactory; +import ai.djl.util.RandomUtils; +import cn.hutool.core.codec.Base64; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.PolygonLabel; +import org.opencv.core.Mat; + +import java.awt.image.BufferedImage; +//import java.awt.image.ColorConvertOp; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * 图片处理工具类 + */ +public class ImageUtils { + + + /** + * 保存DJL图片 + * + * @param img + * @param name + * @param path + */ + public static void save(Image img, String name, String path) { + Path outputDir = Paths.get(path); + Path imagePath = outputDir.resolve(name); + // OpenJDK 不能保存 jpg 图片的 alpha channel + try { + img.save(Files.newOutputStream(imagePath), "png"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 获取图片矩阵BGR + * + * @param img + * @return + */ + public static byte[] getMatrixBGR(Image img) { + if (img.getWrappedImage() instanceof BufferedImage){ + return BufferedImageUtils.getMatrixBGR((BufferedImage)img.getWrappedImage()); + }else if (img.getWrappedImage() instanceof Mat){ + return OpenCVUtils.getMatrixBGR((Mat)img.getWrappedImage()); + }else { + throw new RuntimeException("不支持的图片类型"); + } + } + + + /** + * 保存图片,含检测框 + * + * @param img + * @param detection + * @param name + * @param path + * @throws IOException + */ + public static void saveBoundingBoxImage( + Image img, DetectedObjects detection, String name, String path) throws IOException { + // Make image copy with alpha channel because original image was jpg + img.drawBoundingBoxes(detection); + Path outputDir = Paths.get(path); + Files.createDirectories(outputDir); + Path imagePath = outputDir.resolve(name); + // OpenJDK can't save jpg with alpha channel + img.save(Files.newOutputStream(imagePath), "png"); + } + + + + /** + * 计算左上角，右下角坐标 （x0,y0,x1,y1） + * Get absolute coordinations + * + * @param rect + * @param width + * @param height + * @return + */ + public static int[] rectXYXY(ai.djl.modality.cv.output.Rectangle rect, int width, int height) { + int left = Math.max((int) (width * rect.getX()), 0); + int top = Math.max((int) (height * rect.getY()), 0); + int right = Math.min((int) (width * (rect.getX() + rect.getWidth())), width - 1); + int bottom = Math.min((int) (height * (rect.getY() + rect.getHeight())), height - 1); + return new int[] {left, top, right, bottom}; + } + + /** + * 列出文件夹下的所有图片文件 + * List all image files under the folder + * + * @param folderPath + * @return + */ + public static List listImageFiles(String folderPath) { + File folder = new File(folderPath); + List imageFiles = new ArrayList<>(); + if (folder.exists() && folder.isDirectory()) { + File[] files = folder.listFiles(); + if (files == null) { + return imageFiles; + } + for (File file : files) { + if (file.isFile()) { + String name = file.getName().toLowerCase(); + if (name.endsWith(".jpg") || name.endsWith(".jpeg") || + name.endsWith(".png") || name.endsWith(".bmp") || + name.endsWith(".gif") || name.endsWith(".tiff") || + name.endsWith(".webp")) { + imageFiles.add(file); + } + } + } + } + return imageFiles; + } + + /** + * 读取指定目录下所有图片，返回 List（DJL 格式） + * + * @param folderPath 图片文件夹路径 + * @return List + * @throws IOException + */ + public static List readImagesFromFolder(String folderPath) throws IOException { + File folder = new File(folderPath); + List imageList = new ArrayList<>(); + if (folder.exists() && folder.isDirectory()) { + File[] files = folder.listFiles(); + if (files == null) { + return imageList; + } + for (File file : files) { + if (file.isFile()) { + String name = file.getName().toLowerCase(); + if (name.endsWith(".jpg") || name.endsWith(".jpeg") || + name.endsWith(".png") || name.endsWith(".bmp") || + name.endsWith(".gif") || name.endsWith(".tiff") || + name.endsWith(".webp")) { + + Image img = SmartImageFactory.getInstance().fromInputStream(Files.newInputStream(file.toPath())); + imageList.add(img); + } + } + } + } + return imageList; + } + + /** + * 判断所有图片尺寸是否一致 + * + * @param images 图片列表 + */ + public static boolean isAllImageSizeEqual(List images) { + if (images == null || images.isEmpty()) { + return true; // 空集合视为一致 + } + int width = images.get(0).getWidth(); + int height = images.get(0).getHeight(); + for (Image img : images) { + if (img.getWidth() != width || img.getHeight() != height) { + return false; + } + } + return true; + } + + + + /** + * 拷贝图片 + * @param src + * @return + */ + public static Image copy(Image src) { + Object srcData = src.getWrappedImage(); + //当图片是BufferedImage，DJL的duplicate会有问题 + if (srcData instanceof BufferedImage) { + return SmartImageFactory.getInstance().fromBufferedImage(BufferedImageUtils.copyBufferedImage((BufferedImage) srcData)); + }else{ + return src.duplicate(); + } + } + + /** + * 为不同分类生成不同颜色 + * @param background + * @param opacity + * @param classes + * @return + */ + public static int[] generateColors(int background, int opacity, List classes) { + int[] colors = new int[classes.size()]; + colors[0] = background; + for (int i = 1; i < classes.size(); i++) { + int red = RandomUtils.nextInt(256); + int green = RandomUtils.nextInt(256); + int blue = RandomUtils.nextInt(256); + colors[i] = opacity << 24 | red << 16 | green << 8 | blue; + } + return colors; + } + + /** + * 生成不同颜色遮罩 + * @param colors + * @param mask + * @return + */ + public static Image getColorOverlay(int[] colors,int[][] mask) { + int height = mask.length; + int width = mask[0].length; + int[] pixels = new int[width * height]; + for (int h = 0; h < height; h++) { + for (int w = 0; w < width; w++) { + int index = mask[h][w]; + pixels[h * width + w] = colors[index]; + } + } + return SmartImageFactory.getInstance().fromPixels(pixels, width, height); + } + + /** + * 绘制遮罩 + * @param categoryMask + * @param image + * @param opacity + * @param background + */ + public static void drawMask(CategoryMask categoryMask, Image image, int opacity, int background) { + int[] colors = generateColors(background, opacity, categoryMask.getClasses()); + Image maskImage = getColorOverlay(colors, categoryMask.getMask()); + image.drawImage(maskImage, true); + } + + /** + * 保存 Image 到指定路径，格式根据后缀自动推断 + */ + public static void save(Image image, Path path) throws IOException { + String fileName = path.getFileName().toString().toLowerCase(); + String format = "png"; // 默认 png + if (fileName.endsWith(".jpg") || fileName.endsWith(".jpeg")) { + format = "jpg"; + } else if (fileName.endsWith(".bmp")) { + format = "bmp"; + } else if (fileName.endsWith(".webp")) { + format = "webp"; + } + Files.createDirectories(path.getParent()); + try (OutputStream os = Files.newOutputStream(path)) { + image.save(os, format); + } + } + + /** + * 保存 Image 到指定路径，格式根据后缀自动推断 + */ + public static void save(Image image, Path path, String format) throws IOException { + Files.createDirectories(path.getParent()); + try (OutputStream os = Files.newOutputStream(path)) { + image.save(os, format); + } + } + + /** + * 保存 Image 到指定路径 + */ + public static void save(Image image, String imagePath) throws IOException { + Path path = Paths.get(imagePath); + Files.createDirectories(path.getParent()); + try (OutputStream os = Files.newOutputStream(path)) { + image.save(os, "png"); + } + } + + /** + * 转换为 BufferedImage + */ + public static BufferedImage toBufferedImage(Image image) { + Object wrapped = image.getWrappedImage(); + if (wrapped instanceof BufferedImage) { + return (BufferedImage) wrapped; + } else if (wrapped instanceof Mat) { + Mat mat = (Mat) wrapped; + return OpenCVUtils.mat2Image(mat); + } else { + throw new IllegalArgumentException("Unsupported wrapped image type: " + wrapped.getClass()); + } + } + + /** + * 转换为 Mat + */ + public static Mat toMat(Image image) { + Object wrapped = image.getWrappedImage(); + if (wrapped instanceof BufferedImage) { + return OpenCVUtils.image2Mat((BufferedImage) wrapped); + } else if (wrapped instanceof Mat) { + return (Mat) wrapped; + } else { + throw new IllegalArgumentException("Unsupported wrapped image type: " + wrapped.getClass()); + } + } + + /** + * Image 转 byte[] （默认 png 格式） + */ + public static byte[] toBytes(Image image, String format) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + image.save(baos, format); + return baos.toByteArray(); + } catch (IOException e) { + throw new RuntimeException("Failed to convert Image to byte[]", e); + } + } + + /** + * 保存 Image 到 OutputStream + * + * @param image 图像对象 + * @param os 输出流（需要调用方负责关闭） + * @param format 保存格式（png/jpg/webp） + */ + public static void toOutputStream(Image image, OutputStream os, String format) { + try { + image.save(os, format); + } catch (IOException e) { + throw new RuntimeException("Failed to write image to OutputStream", e); + } + } + + /** + * 转换 Image 为 Base64 字符串 + * + * @param image 图像对象 + * @param format 输出格式（png/jpg/webp） + * @return Base64 编码的字符串 + */ + public static String toBase64(Image image, String format) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + image.save(baos, format); + return Base64.encode(baos.toByteArray()); + } catch (IOException e) { + throw new RuntimeException("Failed to convert image to Base64", e); + } + } + + /** + * 释放 OpenCV Mat + * @param image + */ + public static void releaseOpenCVMat(Image image){ + if (image != null && image.getWrappedImage() instanceof Mat){ + ((Mat)image.getWrappedImage()).release(); + } + } + + /** + * 绘制检测结果 + * @param sourceImage + * @param detectionResponse + * @return + */ + public static Image drawBoundingBoxes(Image sourceImage, DetectionResponse detectionResponse){ + Object srcData = sourceImage.getWrappedImage(); + if (srcData instanceof BufferedImage) { + BufferedImage copyBufferedImage = BufferedImageUtils.copyBufferedImage((BufferedImage) srcData); + BufferedImageUtils.drawBoundingBoxes(copyBufferedImage, detectionResponse); + return SmartImageFactory.getInstance().fromBufferedImage(copyBufferedImage); + }else if (srcData instanceof Mat) { + Mat srcMat = ((Mat) srcData).clone(); + OpenCVUtils.drawBoundingBoxes(srcMat, detectionResponse); + return SmartImageFactory.getInstance().fromMat(srcMat); + }else { + throw new IllegalArgumentException("Unsupported wrapped image type: " + srcData.getClass()); + } + } + + /** + * 逆时针旋转图片 + * + * @param image + * @param times + * @return + */ + public static Image rotateImg(Image image, int times) { + try (NDManager manager = NDManager.newBaseManager()) { + NDArray rotated = NDImageUtils.rotate90(image.toNDArray(manager), times); + return OpenCVImageFactory.getInstance().fromNDArray(rotated); + } + } + + /** + * 图片旋转 + * + * @param manager + * @param image + * @return + */ + public static Image rotateImg(NDManager manager, Image image) { + NDArray rotated = NDImageUtils.rotate90(image.toNDArray(manager), 1); + return ImageFactory.getInstance().fromNDArray(rotated); + } + + + public static void drawPolygonWithText(Image image, List polygonLabelList, int fontSize) { + Object srcData = image.getWrappedImage(); + if (srcData instanceof BufferedImage) { + BufferedImageUtils.drawPolygonWithText((BufferedImage) srcData, polygonLabelList, fontSize); + }else if (srcData instanceof Mat) { + Mat srcMat = (Mat) srcData; + OpenCVUtils.drawPolygonWithText(srcMat, polygonLabelList, fontSize); + }else { + throw new IllegalArgumentException("Unsupported wrapped image type: " + srcData.getClass()); + } + } + + + + /** + * 绘制矩形框 + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(Image image, DetectionRectangle box, String text) { + Object srcData = image.getWrappedImage(); + if (srcData instanceof BufferedImage) { + BufferedImageUtils.drawRectAndText((BufferedImage) srcData, box, text,12); + }else if (srcData instanceof Mat) { + Mat srcMat = (Mat) srcData; + OpenCVUtils.drawRectAndText(srcMat, box, text, 0.5); + }else { + throw new IllegalArgumentException(); + } + } + + /** + * 绘制矩形框 + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(Image image, DetectionRectangle box, String text, double fontSize) { + Object srcData = image.getWrappedImage(); + if (srcData instanceof BufferedImage) { + BufferedImageUtils.drawRectAndText((BufferedImage) srcData, box, text, (int)fontSize); + }else if (srcData instanceof Mat) { + Mat srcMat = (Mat) srcData; + OpenCVUtils.drawRectAndText(srcMat, box, text, fontSize); + }else { + throw new IllegalArgumentException(); + } + } + + public static void drawRectAndText(Image image, DetectionInfo detectionInfo){ + Object srcData = image.getWrappedImage(); + if (srcData instanceof BufferedImage) { + BufferedImageUtils.drawRectAndText((BufferedImage) srcData, detectionInfo); + }else if (srcData instanceof Mat) { + Mat srcMat = (Mat) srcData; + OpenCVUtils.drawRectAndText(srcMat, detectionInfo); + }else { + throw new IllegalArgumentException(); + } + } + + public static void drawRectAndText(Image image, List detectionInfoList){ + for(DetectionInfo detectionInfo : detectionInfoList){ + drawRectAndText(image, detectionInfo); + } + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java similarity index 52% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java index 678251b..89eaa93 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/LetterBoxUtils.java @@ -1,5 +1,8 @@ package cn.smartjavaai.common.utils; +import ai.djl.modality.cv.output.Landmark; +import ai.djl.modality.cv.output.Point; +import ai.djl.modality.cv.output.Rectangle; import ai.djl.modality.cv.util.NDImageUtils; import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDManager; @@ -7,7 +10,9 @@ import ai.djl.ndarray.index.NDIndex; import ai.djl.ndarray.types.DataType; import ai.djl.ndarray.types.Shape; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * 按比例缩放，剩余空间用指定颜色填充 @@ -60,7 +65,7 @@ public class LetterBoxUtils { // NDArray paddingImg = manager // .full(new Shape(targetW, targetH, 3), padColor, DataType.UINT8); - NDArray paddingImg = manager.zeros(new Shape(targetW, targetH, 3), DataType.FLOAT32); + NDArray paddingImg = manager.zeros(new Shape(targetH, targetW, 3), DataType.FLOAT32); paddingImg = paddingImg.add(114); int padW = targetW - newW; @@ -121,4 +126,88 @@ public class LetterBoxUtils { return boxes; } + /** + * 恢复缩放后的 box(左上角坐标) + * @param rectangle + * @param scale + * @param origImageWidth + * @param origImageHeight + */ + public static Rectangle restoreBox(Rectangle rectangle, float scale, int origImageWidth, int origImageHeight, int inputWidth, int inputHeight){ + double paddingWidth = (inputWidth - origImageWidth * scale) / 2; + double paddingHeight = (inputHeight - origImageHeight * scale) / 2; + + // 去掉 padding + double x_noPad = rectangle.getX() - paddingWidth; + double y_noPad = rectangle.getY() - paddingHeight; + + //模型输出就是原图坐标 + double x1 = x_noPad / scale / origImageWidth; + double y1 = y_noPad / scale / origImageHeight; + double boxW = rectangle.getWidth() / scale / origImageWidth ; + double boxH = rectangle.getHeight() / scale / origImageHeight; + return new Rectangle(x1, y1, boxW, boxH); + } + + /** + * 恢复缩放后的 box(左上角坐标) + * @param landmark + * @param scale + * @param origImageWidth + * @param origImageHeight + */ + public static Landmark restoreBox(Landmark landmark, float scale, int origImageWidth, int origImageHeight, int inputWidth, int inputHeight, boolean isNormalized){ + double x = 0; + double y = 0; + double width = 0; + double height = 0; + if(isNormalized){ + x = landmark.getX() * inputWidth; + y = landmark.getY() * inputHeight; + width = landmark.getWidth() * inputWidth; + height = landmark.getHeight() * inputHeight; + }else{ + x = landmark.getX(); + y = landmark.getY(); + width = landmark.getWidth(); + height = landmark.getHeight(); + } + double paddingWidth = (inputWidth - origImageWidth * scale) / 2; + double paddingHeight = (inputHeight - origImageHeight * scale) / 2; + + // 去掉 padding + double x_noPad = x - paddingWidth; + double y_noPad = y - paddingHeight; + + //模型输出就是原图坐标 + double x1 = x_noPad / scale / origImageWidth; + double y1 = y_noPad / scale / origImageHeight; + double boxW = width / scale / origImageWidth ; + double boxH = height / scale / origImageHeight; + + List points = new ArrayList<>(); + // 要求关键点未归一化 + landmark.getPath().forEach(point -> { + double pointX = (point.getX() - paddingWidth) / scale; + double pointY = (point.getY() - paddingHeight) / scale; + points.add(new Point(pointX, pointY)); + }); + return new Landmark(x1, y1, boxW, boxH, points); + } + + /** + * 获取缩放后的图片大小 + * @param origW 原始图片宽度 + * @param origH 原始图片高度 + * @param targetWidth 目标图片宽度 + * @param targetHeight 目标图片高度 + * @return + */ + public static int[] getResizeSize(int origW, int origH, int targetWidth, int targetHeight){ + float r = Math.min(targetWidth / (float) origW, targetHeight / (float) origH); + int newW = Math.round(origW * r); + int newH = Math.round(origH * r); + return new int[]{newW, newH}; + } + } diff --git a/common/src/main/java/cn/smartjavaai/common/utils/NMSUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/NMSUtils.java new file mode 100644 index 0000000..35bbd2a --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/NMSUtils.java @@ -0,0 +1,177 @@ +package cn.smartjavaai.common.utils; + +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDArrays; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author dwj + * @date 2025/7/23 + */ +public class NMSUtils { + + /** + * 通用 NMS 方法，输入 NDArray 形式的 boxes 和 scores，返回保留的索引列表 + * + * @param boxes NDArray 形状为 (N, 4)，格式为 [x1, y1, x2, y2] + * @param scores NDArray 形状为 (N,) 或 (N,1)，每个 box 的置信度 + * @param iouThreshold IOU 阈值，超过该阈值则认为有重叠 + * @return 保留框的索引列表 + */ + public static int[] nms(NDArray boxes, NDArray scores, float iouThreshold) { + if (boxes.isEmpty()) { + return new int[0]; + } + + NDArray x1 = boxes.get(":, 0"); + NDArray y1 = boxes.get(":, 1"); + NDArray x2 = boxes.get(":, 2"); + NDArray y2 = boxes.get(":, 3"); + + NDArray areas = x2.sub(x1).add(1).mul(y2.sub(y1).add(1)); + + // 按照置信度降序排序 + NDArray order = scores.argSort().flip(0); + + List keep = new ArrayList<>(); + + while (order.size() > 0) { + int idx = (int)order.getLong(0); + keep.add(idx); + + if (order.size() == 1) break; + + NDArray currentBox = boxes.get(idx); + NDArray others = boxes.get(order); + + NDArray xx1 = x1.get(order).maximum(x1.get(idx)); + NDArray yy1 = y1.get(order).maximum(y1.get(idx)); + NDArray xx2 = x2.get(order).minimum(x2.get(idx)); + NDArray yy2 = y2.get(order).minimum(y2.get(idx)); + + NDArray w = xx2.sub(xx1).add(1).maximum(0); + NDArray h = yy2.sub(yy1).add(1).maximum(0); + NDArray inter = w.mul(h); + + NDArray remAreas = areas.get(order); + NDArray union = remAreas.add(areas.get(idx)).sub(inter); + NDArray iou = inter.div(union); + + NDArray mask = iou.lte(iouThreshold); + order = order.get(mask); + } + return keep.stream().mapToInt(i -> i).toArray(); + } + + + + /** + * 批量执行 NMS，输入 NDArray 形式的 boxes、scores 和 idxs，返回保留的索引列表 + * + * @param boxes NDArray 形状为 (N, 4)，格式为 [x1, y1, x2, y2] + * @param scores NDArray 形状为 (N,) 或 (N,1)，每个 box 的置信度 + * @param idxs NDArray 形状为 (N,)，每个 box 对应的 batch id + * @param iouThreshold IOU 阈值，超过该阈值则认为有 + * @return 批量保留框的索引列表 + * + */ + public static NDArray batchedNms(NDArray boxes, NDArray scores, NDArray idxs, float iouThreshold, NDManager manager) { + +// System.out.println("---------------boxes:" + Arrays.toString(boxes.toFloatArray())); + + List keepList = new ArrayList<>(); + + // 获取唯一 batch id + NDArray uniqueIdxs = idxs.unique().get(0); + + for (long batchId : uniqueIdxs.toLongArray()) { + // 找出当前 batch 的框 + NDArray mask = idxs.eq(batchId); + NDArray batchBoxes = boxes.get(mask); + NDArray batchScores = scores.get(mask); + + // 执行单 batch NMS + int[] keepIndices = mtcnnNms(batchBoxes, batchScores, iouThreshold); + + if (keepIndices.length > 0) { + // 将局部索引映射回全局索引 + NDArray globalIndices = manager.arange(boxes.getShape().get(0)) + .get(mask) + .toType(DataType.INT64, false) + .get(manager.create(keepIndices)); + + keepList.add(globalIndices); + } + } + + if (keepList.isEmpty()) { + return manager.create(new long[0]); + } + return NDArrays.concat(new NDList(keepList)); + } + + + public static int[] mtcnnNms(NDArray boxes, NDArray scores, float iouThreshold) { + if (boxes.isEmpty()) { + return new int[0]; + } + + NDArray x1 = boxes.get(":, 0"); + NDArray y1 = boxes.get(":, 1"); + NDArray x2 = boxes.get(":, 2"); + NDArray y2 = boxes.get(":, 3"); + + // 面积 + NDArray areas = x2.sub(x1).add(1).mul(y2.sub(y1).add(1)); + + // scores 降序索引 + NDArray order = scores.argSort(); + //System.out.println("order：" + order.getShape()); + //System.out.println("order：" + Arrays.toString(order.toLongArray())); + + List keep = new ArrayList<>(); + + while (order.size() > 0) { + int i = (int) order.getLong(-1); + keep.add(i); + + if (order.size() == 1) break; // 没框了就退出 + + // 剩余框 + NDArray idx = order.get("0:-1"); + + NDArray xx1 = x1.get(i).maximum(x1.get(idx)); + NDArray yy1 = y1.get(i).maximum(y1.get(idx)); + NDArray xx2 = x2.get(i).minimum(x2.get(idx)); + NDArray yy2 = y2.get(i).minimum(y2.get(idx)); + + NDArray w = xx2.sub(xx1).add(1).maximum(0); + NDArray h = yy2.sub(yy1).add(1).maximum(0); + NDArray inter = w.mul(h); + + NDArray union = areas.get(i).minimum(areas.get(idx)); + NDArray iou = inter.div(union); + +// System.out.println("Max IoU: " + iou.max().getFloat()); +// System.out.println("Min IoU: " + iou.min().getFloat()); +// System.out.println("Mean IoU: " + iou.mean().getFloat()); + +// System.out.println("Before: " + order.size()); + // 保留 IoU <= 阈值的框 + NDArray mask = iou.lte(iouThreshold); +// System.out.println("Mask size: " + mask.size() + " True count: " + mask.sum()); + + // 更新 order + order = idx.get(mask); +// System.out.println("After: " + order.size()); + } + + return keep.stream().mapToInt(Integer::intValue).toArray(); + } + +} diff --git a/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java new file mode 100644 index 0000000..bec5fb8 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java @@ -0,0 +1,751 @@ +package cn.smartjavaai.common.utils; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.opencv.OpenCVImageFactory; +import ai.djl.util.RandomUtils; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.PolygonLabel; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.HeadPose; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencv.core.*; +import org.opencv.core.Point; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; + +import java.awt.*; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * OpenCV 工具类 + */ +public class OpenCVUtils { + + + /** + * canny算法，边缘检测 + * + * @param src + * @return + */ + public static Mat canny(Mat src) { + Mat mat = src.clone(); + Imgproc.Canny(src, mat, 100, 200); + return mat; + } + + /** + * 画线 + * + * @param mat + * @param point1 + * @param point2 + */ + public static void line(Mat mat, Point point1, Point point2) { + Imgproc.line(mat, point1, point2, new Scalar(255, 255, 255), 1); + } + + /** + * NDArray to opencv_core.Mat + * + * @param manager + * @param srcPoints + * @param dstPoints + * @return + */ + public static Mat toOpenCVMat(NDManager manager, NDArray srcPoints, NDArray dstPoints) { + NDArray svdMat = SVDUtils.transformationFromPoints(manager, srcPoints, dstPoints); + double[] doubleArray = svdMat.toDoubleArray(); + Mat newSvdMat = new Mat(2, 3, CvType.CV_64F); + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 3; j++) { + newSvdMat.put(i, j, doubleArray[i * 3 + j]); + } + } + return newSvdMat; + } + + /** + * double[][] points array to Mat + * @param points + * @return + */ + public static Mat toOpenCVMat(double[][] points) { + Mat mat = new Mat(5, 2, CvType.CV_64F); + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 2; j++) { + mat.put(i, j, points[i * 5 + j]); + } + } + return mat; + } + + /** + * 变换矩阵的逆矩阵 + * + * @param src + * @return + */ + public static Mat invertAffineTransform(Mat src) { + Mat dst = src.clone(); + Imgproc.invertAffineTransform(src, dst); + return dst; + } + + /** + * Mat to BufferedImage + * + * @param mat + * @return + */ + public static BufferedImage mat2Image(Mat mat) { + int width = mat.width(); + int height = mat.height(); + byte[] data = new byte[width * height * (int) mat.elemSize()]; + Imgproc.cvtColor(mat, mat, 4); + mat.get(0, 0, data); + BufferedImage ret = new BufferedImage(width, height, 5); + ret.getRaster().setDataElements(0, 0, width, height, data); + return ret; + } + + /** + * BufferedImage to Mat + * + * @param img + * @return + */ + public static Mat image2Mat(BufferedImage img) { + int width = img.getWidth(); + int height = img.getHeight(); + + // 强制转换为 TYPE_3BYTE_BGR，自动去除透明通道 + BufferedImage convertedImg = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR); + Graphics2D g2d = convertedImg.createGraphics(); + g2d.drawImage(img, 0, 0, null); + g2d.dispose(); + + byte[] data = ((DataBufferByte) convertedImg.getRaster().getDataBuffer()).getData(); + Mat mat = new Mat(height, width, CvType.CV_8UC3); + mat.put(0, 0, data); + return mat; + } + + /** + * 透视变换 + * + * @param src + * @param srcPoints + * @param dstPoints + * @return + */ + public static Mat perspectiveTransform(Mat src, Mat srcPoints, Mat dstPoints) { + Mat dst = src.clone(); + Mat warp_mat = Imgproc.getPerspectiveTransform(srcPoints, dstPoints); + Imgproc.warpPerspective(src, dst, warp_mat, dst.size()); + warp_mat.release(); + return dst; + } + + /** + * 绘制矩形框和文字 + * + * @param image + * @param detectionInfoList + */ + public static void drawRectAndText(Mat image, List detectionInfoList) { + if(CollectionUtils.isEmpty(detectionInfoList)) + return; + for(DetectionInfo detectionInfo : detectionInfoList){ + drawRectAndText(image, detectionInfo); + } + } + + + /** + * 绘制矩形框和文字 + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(Mat image, DetectionRectangle box, String text) { + if(image.empty()) + return; + int x = box.getX(); + int y = box.getY(); + int width = box.getWidth(); + int height = box.getHeight(); + Scalar rectangleColor = new Scalar((double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178)); + // 绘制矩形框 + Point pt1 = new Point(x, y); + Point pt2 = new Point(x + width, y + height); + Imgproc.rectangle(image, pt1, pt2, rectangleColor, 2); + Scalar textColor = new Scalar(255.0, 255.0, 255.0); + putTextWithBackground(image, text, pt1, textColor, rectangleColor, 1); + } + + /** + * 绘制矩形框和文字 + * @param image + * @param box + * @param text + */ + public static void drawRectAndText(Mat image, DetectionRectangle box, String text, double fontSize) { + if(image.empty()) + return; + int x = box.getX(); + int y = box.getY(); + int width = box.getWidth(); + int height = box.getHeight(); + Scalar rectangleColor = new Scalar((double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178)); + // 绘制矩形框 + Point pt1 = new Point(x, y); + Point pt2 = new Point(x + width, y + height); + Imgproc.rectangle(image, pt1, pt2, rectangleColor, 2); + Scalar textColor = new Scalar(255.0, 255.0, 255.0); + putTextWithBackground(image, text, pt1, textColor, rectangleColor, 1, fontSize); + } + + + /** + * 绘制矩形框和文字 + * + * @param image + * @param detectionInfo + */ + public static void drawRectAndText(Mat image, DetectionInfo detectionInfo) { + if (image == null) return; + int x = detectionInfo.getDetectionRectangle().getX(); + int y = detectionInfo.getDetectionRectangle().getY(); + int width = detectionInfo.getDetectionRectangle().getWidth(); + int height = detectionInfo.getDetectionRectangle().getHeight(); + Scalar rectangleColor = new Scalar((double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178), (double)RandomUtils.nextInt(178)); + // 绘制矩形框 + Point pt1 = new Point(x, y); + Point pt2 = new Point(x + width, y + height); + Imgproc.rectangle(image, pt1, pt2, rectangleColor, 2); + // 绘制文字 + String className = null; + Scalar textColor = new Scalar(255.0, 255.0, 255.0); + //目标检测信息 + if(detectionInfo.getObjectDetInfo() != null){ + className = detectionInfo.getObjectDetInfo().getClassName(); + putTextWithBackground(image, className, pt1, textColor, rectangleColor, 1); + } + //人脸 + if(detectionInfo.getFaceInfo() != null){ + className = "face"; + putTextWithBackground(image, className, pt1, textColor, rectangleColor, 1); + //绘制关键点 + drawLandmarks(image, detectionInfo.getFaceInfo().getKeyPoints()); + //绘制人脸属性 + if(detectionInfo.getFaceInfo().getFaceAttribute() != null){ + drawFaceAttribute(detectionInfo.getFaceInfo().getFaceAttribute(), detectionInfo.getDetectionRectangle(), image); + } + } + } + + + /** + * 在Mat上绘制矩形框和文字 + * + * @param mat 待绘制的Mat + * @param x 矩形左上角X + * @param y 矩形左上角Y + * @param width 矩形宽度 + * @param height 矩形高度 + * @param color 框的颜色，例如 new Scalar(0, 255, 0) 绿色 + * @param thickness 框线宽度 + * @param text 需要绘制的文字，可以为null或空 + * @param fontScale 文字缩放比例 + * @param textColor 文字颜色 + */ + public static void drawRectAndText(Mat mat, + int x, int y, int width, int height, + Scalar color, int thickness, + String text, double fontScale, Scalar textColor) { + + if (mat == null || mat.empty()) return; + + // 绘制矩形框 + Point pt1 = new Point(x, y); + Point pt2 = new Point(x + width, y + height); + Imgproc.rectangle(mat, pt1, pt2, color, thickness); + + // 绘制文字 + if (text != null && !text.isEmpty()) { + int baseline[] = new int[1]; + Size textSize = Imgproc.getTextSize(text, Imgproc.FONT_HERSHEY_SIMPLEX, fontScale, thickness, baseline); + // 保证文字不超出矩形 + Point textOrg = new Point(x, y - 5 < 0 ? y + textSize.height + 5 : y - 5); + Imgproc.putText(mat, text, textOrg, Imgproc.FONT_HERSHEY_SIMPLEX, fontScale, textColor, thickness); + } + } + + /** + * 将 Bytedeco 的 Mat 转换为 OpenCV 官方的 Mat + * @param src Bytedeco Mat (BGR 或 BGRA) + * @return OpenCV Mat (BGR 或 BGRA) + */ +// public static org.opencv.core.Mat convertToOpenCVMat(org.bytedeco.opencv.opencv_core.Mat bMat) { +// +// +// try { +// int width = bMat.cols(); +// int height = bMat.rows(); +// int channels = bMat.channels(); +// +// // 创建 OpenCV Mat +// org.opencv.core.Mat cvMat = new org.opencv.core.Mat(height, width, channels == 3 ? CvType.CV_8UC3 : CvType.CV_8UC1); +// +// // 从 bytedeco Mat 获取像素数据 +// byte[] data = new byte[width * height * channels]; +// bMat.data().get(data); +// +// // 填充到 OpenCV Mat +// cvMat.put(0, 0, data); +// return cvMat; +// } catch (Throwable e) { +// e.printStackTrace(); +// } +// return null; +// } + + + /** + * 从 OpenCV Mat 中获取 BGR 格式矩阵数据 + * + * @param mat OpenCV Mat，需为 CV_8UC3 或可转换为 BGR 格式 + * @return BGR 格式字节数组，按行连续存储 + */ + public static byte[] getMatrixBGR(Mat mat) { + if (mat == null || mat.empty()) { + throw new IllegalArgumentException("Mat 不能为空"); + } + + // 确保是三通道 BGR 格式 + Mat bgrMat = new Mat(); + if (mat.channels() == 3) { + mat.copyTo(bgrMat); + } else if (mat.channels() == 4) { + // RGBA 转 BGR + Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_RGBA2BGR); + } else if (mat.channels() == 1) { + // 灰度转 BGR + Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_GRAY2BGR); + } else { + throw new IllegalArgumentException("不支持的通道数: " + mat.channels()); + } + + int size = (int) (bgrMat.total() * bgrMat.channels()); + byte[] data = new byte[size]; + bgrMat.get(0, 0, data); + bgrMat.release(); // 释放临时 Mat + return data; + } + + /** + * 在图像上绘制带白色背景、黑色文字的文本 + */ + public static void putTextWithBackground(Mat image, String text, org.opencv.core.Point origin, Scalar textColor, Scalar backgroundColor, int padding) { + // 默认字体 + int font = Imgproc.FONT_HERSHEY_SIMPLEX; + // 默认字体缩放大小 + double fontScale = 0.6; + //线条粗细 + int thickness = 2; + //获取文字大小 + int[] baseLine = new int[1]; + Size textSize = Imgproc.getTextSize(text, font, fontScale, thickness, baseLine); + int textWidth = (int) textSize.width; + int textHeight = (int) textSize.height; + + //计算带padding的背景框 + org.opencv.core.Point bgTopLeft = new org.opencv.core.Point(origin.x - padding, origin.y - textHeight - padding); + org.opencv.core.Point bgBottomRight = new org.opencv.core.Point(origin.x + textWidth + padding, origin.y + baseLine[0] + padding); + + //绘制背景矩形 + Imgproc.rectangle(image, bgTopLeft, bgBottomRight, backgroundColor, Imgproc.FILLED); + + //绘制文字（黑色） + Imgproc.putText(image, text, origin, font, fontScale, textColor, thickness); + } + + /** + * 在图像上绘制带白色背景、黑色文字的文本 + */ + public static void putTextWithBackground(Mat image, String text, org.opencv.core.Point origin, Scalar textColor, Scalar backgroundColor, int padding, double fontScale) { + // 默认字体 + int font = Imgproc.FONT_HERSHEY_SIMPLEX; + //线条粗细 + int thickness = 1; + //获取文字大小 + int[] baseLine = new int[1]; + Size textSize = Imgproc.getTextSize(text, font, fontScale, thickness, baseLine); + int textWidth = (int) textSize.width; + int textHeight = (int) textSize.height; + + //计算带padding的背景框 + org.opencv.core.Point bgTopLeft = new org.opencv.core.Point(origin.x - padding, origin.y - textHeight - padding); + org.opencv.core.Point bgBottomRight = new org.opencv.core.Point(origin.x + textWidth + padding, origin.y + baseLine[0] + padding); + + //绘制背景矩形 + Imgproc.rectangle(image, bgTopLeft, bgBottomRight, backgroundColor, Imgproc.FILLED); + + //绘制文字（黑色） + Imgproc.putText(image, text, origin, font, fontScale, textColor, thickness); + } + + /** + * 绘制检测结果 + * @param image 待绘制的图片 + * @param detectionResponse 检测结果 + * @return 绘制后的图片 + */ + public static void drawBoundingBoxes(Mat image, DetectionResponse detectionResponse){ + if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){ + throw new IllegalArgumentException("无目标数据"); + } + drawRectAndText(image, detectionResponse.getDetectionInfoList()); + } + + /** + * 在 Mat 上绘制关键点 + * @param mat OpenCV 图像 + * @param keyPoints 人脸关键点列表 + */ + public static void drawLandmarks(Mat mat, List keyPoints) { + // 设置颜色 (BGR 格式)，这里是橙色 (0,96,246) + Scalar color = new Scalar(0, 96, 246); + // 遍历关键点，用圆来表示 (比矩形更自然) + for (cn.smartjavaai.common.entity.Point p : keyPoints) { + Point cvPoint = new Point(p.getX(), p.getY()); + Imgproc.circle(mat, cvPoint, 2, color, 2, Imgproc.LINE_AA, 0); + } + } + + /** + * 在 Mat 上绘制多行文字，并带背景 + * @param mat OpenCV Mat + * @param lines 文字行 + * @param x 起始 x + * @param y 起始 y + */ + public static void drawMultilineTextWithBackground(Mat mat, List lines, int x, int y) { + int fontFace = Imgproc.FONT_HERSHEY_SIMPLEX; + double fontScale = 0.5; // 字体大小 + int thickness = 1; + int baseline[] = {0}; + + // 逐行计算最大宽度 & 总高度 + int maxWidth = 0; + int lineHeight = 0; + for (String line : lines) { + Size textSize = Imgproc.getTextSize(line, fontFace, fontScale, thickness, baseline); + maxWidth = Math.max(maxWidth, (int) textSize.width); + lineHeight = Math.max(lineHeight, (int) (textSize.height + baseline[0])); + } + + int padding = 4; + int boxWidth = maxWidth + padding * 2; + int boxHeight = lineHeight * lines.size() + padding * 2; + + // 绘制背景矩形 (半透明黑色在 OpenCV 里不好直接实现，只能画实色或用 addWeighted 合成) + Scalar bgColor = new Scalar(0, 0, 0); // BGR = 黑色 + Point topLeft = new Point(x, y); + Point bottomRight = new Point(x + boxWidth, y + boxHeight); + Imgproc.rectangle(mat, topLeft, bottomRight, bgColor, -1); // -1 表示填充 + + // 逐行绘制文字 (白色) + Scalar textColor = new Scalar(255, 255, 255); + for (int i = 0; i < lines.size(); i++) { + int textY = y + padding + (i + 1) * lineHeight; + Imgproc.putText(mat, lines.get(i), + new Point(x + padding, textY), + fontFace, fontScale, textColor, thickness, Imgproc.LINE_AA, false); + } + } + + /** + * 绘制人脸属性 + * @param faceAttribute + * @param rectangle + * @param mat + */ + public static void drawFaceAttribute(FaceAttribute faceAttribute, DetectionRectangle rectangle, Mat mat){ + List lines = new ArrayList<>(); + if (faceAttribute.getGenderType() != null) { + lines.add("gender: " + faceAttribute.getGenderType()); + } + if (faceAttribute.getAge() != null) { + lines.add("age: " + faceAttribute.getAge()); + } + if (faceAttribute.getWearingMask() != null) { + lines.add("mask: " + (faceAttribute.getWearingMask() ? "yes" : "no")); + } + if (faceAttribute.getLeftEyeStatus() != null && faceAttribute.getRightEyeStatus() != null) { + lines.add("eyes: " + faceAttribute.getLeftEyeStatus() + "/" + faceAttribute.getRightEyeStatus()); + } + if (faceAttribute.getHeadPose() != null) { + HeadPose pose = faceAttribute.getHeadPose(); + String pitch = pose.getPitch() != null ? String.valueOf(pose.getPitch().intValue()) : "-"; + String yaw = pose.getYaw() != null ? String.valueOf(pose.getYaw().intValue()) : "-"; + String roll = pose.getRoll() != null ? String.valueOf(pose.getRoll().intValue()) : "-"; + lines.add("head pose: P=" + pitch + " Y=" + yaw + " R=" + roll); + } + if (!lines.isEmpty()) { + drawMultilineTextWithBackground(mat, lines, rectangle.getX(), rectangle.getY()); // 适当偏移 + } + } + + + /** + * 透视变换 + 裁剪 + * @param srcMat + * @param landMarks + * @return + */ + public static Image transformAndCrop(Mat srcMat, List landMarks){ + if (landMarks == null || landMarks.size() != 4) { + throw new IllegalArgumentException("必须提供4个关键点"); + } + + // 步骤 1：排序为 左上、右上、右下、左下 + List ordered = PointUtils.orderPoints(landMarks); + + ai.djl.modality.cv.output.Point lt = ordered.get(0); + ai.djl.modality.cv.output.Point rt = ordered.get(1); + ai.djl.modality.cv.output.Point rb = ordered.get(2); + ai.djl.modality.cv.output.Point lb = ordered.get(3); + + // 步骤 2：计算目标图像尺寸（宽、高） + int img_crop_width = (int) Math.max( + PointUtils.distance(lt, rt), + PointUtils.distance(rb, lb) + ); + int img_crop_height = (int) Math.max( + PointUtils.distance(lt, lb), + PointUtils.distance(rt, rb) + ); + + // 步骤 3：构造目标坐标点 + List dstPoints = Arrays.asList( + new ai.djl.modality.cv.output.Point(0, 0), + new ai.djl.modality.cv.output.Point(img_crop_width, 0), + new ai.djl.modality.cv.output.Point(img_crop_width, img_crop_height), + new ai.djl.modality.cv.output.Point(0, img_crop_height) + ); + + // 步骤 4：透视变换 + Mat srcPoint2f = DJLCommonUtils.toMat(ordered); + Mat dstPoint2f = DJLCommonUtils.toMat(dstPoints); + Mat cvMat = OpenCVUtils.perspectiveTransform(srcMat, srcPoint2f, dstPoint2f); + + // 步骤 5：转为 DJL Image + 裁剪 + Image subImg = OpenCVImageFactory.getInstance().fromImage(cvMat); + subImg = subImg.getSubImage(0, 0, img_crop_width, img_crop_height); + + // 释放资源 + cvMat.release(); + srcPoint2f.release(); + dstPoint2f.release(); + return subImg; + } + + /** + * Mat To MatOfPoint + * @param mat + * @return + */ + public static MatOfPoint matToMatOfPoint(Mat mat) { + int rows = mat.rows(); + MatOfPoint matOfPoint = new MatOfPoint(); + + List list = new ArrayList<>(); + for (int i = 0; i < rows; i++) { + Point point = new Point((float) mat.get(i, 0)[0], (float) mat.get(i, 1)[0]); + list.add(point); + } + matOfPoint.fromList(list); + + return matOfPoint; + } + + /** + * Mat To double[][] Array + * @param mat + * @return + */ + public static double[][] matToDoubleArray(Mat mat) { + int rows = mat.rows(); + int cols = mat.cols(); + double[][] doubles = new double[rows][cols]; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + doubles[i][j] = mat.get(i, j)[0]; + } + } + return doubles; + } + + /** + * Mat To float[][] Array + * @param mat + * @return + */ + public static float[][] matToFloatArray(Mat mat) { + int rows = mat.rows(); + int cols = mat.cols(); + + float[][] floats = new float[rows][cols]; + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + floats[i][j] = (float) mat.get(i, j)[0]; + } + } + + return floats; + } + + /** + * Mat To byte[][] Array + * @param mat + * @return + */ + public static byte[][] matToUint8Array(Mat mat) { + int rows = mat.rows(); + int cols = mat.cols(); + + byte[][] bytes = new byte[rows][cols]; + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + bytes[i][j] = (byte) mat.get(i, j)[0]; + } + } + + return bytes; + } + + /** + * float[][] Array To Mat + * @param arr + * @return + */ + public static Mat floatArrayToMat(float[][] arr) { + int rows = arr.length; + int cols = arr[0].length; + Mat mat = new Mat(rows, cols, CvType.CV_32F); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + mat.put(i, j, arr[i][j]); + } + } + + return mat; + } + + /** + * byte[][] Array To Mat + * @param arr + * @return + */ + public static Mat uint8ArrayToMat(byte[][] arr) { + int rows = arr.length; + int cols = arr[0].length; + Mat mat = new Mat(rows, cols, CvType.CV_8U); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + mat.put(i, j, arr[i][j]); + } + } + + return mat; + } + + + /** + * 将自定义 Point 列表转换为 OpenCV Point 列表 + * @param pointList 自定义 Point 列表 + * @return OpenCV Point 列表 + */ + public static List toCvPointList(List pointList) { + if (pointList == null) { + return null; + } + return pointList.stream() + .map(p -> new Point(p.getX(), p.getY())) + .collect(Collectors.toList()); + } + + public static void drawPolygonWithText(Mat mat, List polygonLabelList, int fontSize) { + for (PolygonLabel polygonLabel : polygonLabelList){ + List cvPointList = toCvPointList(polygonLabel.getPoints()); + drawPolygonWithText(mat, cvPointList, polygonLabel.getText(), new Scalar(0, 255, 0), 2); + } + } + + + /** + * 在图像上绘制多边形（任意边数） + * + * @param mat 图像 + * @param points 点的列表（至少2个点） + * @param color 颜色 + * @param thickness 线宽 + */ + public static void drawPolygonWithText(Mat mat, List points, String text, Scalar color, int thickness) { + if (points == null || points.size() < 2) { + return; + } + // 连线 + for (int i = 0; i < points.size(); i++) { + Point p1 = points.get(i); + Point p2 = points.get((i + 1) % points.size()); // 最后一个点连回第一个 + Imgproc.line(mat, p1, p2, color, thickness); + } + if(StringUtils.isNotBlank(text)){ + Scalar textScalar = new Scalar(0,0,0); + // 保证文字不超出矩形 + Imgproc.putText(mat, text, points.get(0), Imgproc.FONT_HERSHEY_SIMPLEX, 1, textScalar, thickness); + } + } + + public static Mat getSubImage(Mat image, int x, int y, int w, int h) { + return image.submat(new Rect(x, y, w, h)); + } + + /** + * 从本地路径读取图片并转为 Mat + * + * @param path 图片路径 + * @return Mat 对象 + */ + public static Mat loadImage(String path) { + // 使用 imread 读取 + Mat mat = Imgcodecs.imread(path); + // 判空，避免后续处理时报错 + if (mat.empty()) { + throw new IllegalArgumentException("无法加载图片: " + path); + } + return mat; + } + + + +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/PointUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/PointUtils.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/PointUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/PointUtils.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/PoolUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/PoolUtils.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/PoolUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/PoolUtils.java diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/SVDUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/SVDUtils.java similarity index 100% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/SVDUtils.java rename to common/src/main/java/cn/smartjavaai/common/utils/SVDUtils.java diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java b/common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java similarity index 97% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java rename to common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java index f949fde..f693618 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java @@ -1,6 +1,7 @@ -package cn.smartjavaai.face.utils; +package cn.smartjavaai.common.utils; -import cn.smartjavaai.face.enums.SimilarityType; + +import cn.smartjavaai.common.enums.SimilarityType; /** * 特征相似度计算工具类 diff --git a/examples/README.md b/examples/README.md index 3309623..0d35131 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,44 +1,40 @@ # SmartJavaAI 示例项目说明 -本项目包含多个基于 SmartJavaAI 平台的人脸识别、活体检测及目标检测的示例代码 ## 项目结构 ``` src/main/java/smartai/examples/ -├── face/ 人脸相关示例 -│ ├── attribute/ 人脸属性检测模块 -│ │ └── FaceAttributeDetDemo.java 示例：检测性别、年龄等人脸属性 -│ ├── facerec/ 人脸识别模块（1:1、1:N） -│ │ ├── FaceNetDemo.java 示例：使用 FaceNet 算法做人脸识别 -│ │ ├── GpuFaceDemo.java 示例：使用 GPU 加速的人脸识别 -│ │ ├── LightFaceDemo.java 示例：轻量级人脸识别模型（适用于嵌入式场景） -│ │ ├── RetinaFaceDemo.java 示例：使用 RetinaFace 进行人脸检测 -│ │ └── SeetaFace6Demo.java 示例：集成 SeetaFace6 的人脸识别 -│ └── liveness/ 活体检测模块 -│ ├── LivenessDetDemo.java 示例：基于图像进行活体检测 -├── objectdetection/ 目标检测模块 -│ └── ObjectDetection.java 示例：使用目标检测模型识别图像中的目标 -└── ocr/ OCR文字识别模块 -│ ├── OcrDetectionDemo.java 示例：OCR通用文字检测示例 -│ ├── OcrDirectionDetDemo.java 示例：OCR方向检测示例 -│ └── OcrRecognizeDemo.java 示例：OCR通用文字识别示例 -├── nlp/ NLP -│ ├── translate/ 机器翻译 -│ └── TranslationDemo.java 示例：机器翻译 +├── face-example/ 人脸检测、人脸识别等功能示例 +├── vision-example/ 通用视觉检测示例：目标检测、目标分割、图像分类等 +├── ocr-example/ OCR文字识别、车牌识别等功能示例 +├── translate-example/ 机器翻译功能示例 +├── speech-example/ 语音识别、语音合成功能示例 + ``` -## 快速开始 +本项目在 `examples` 文件夹下提供了多个示例工程，用于演示各功能模块的使用方法： -1. 克隆本项目 -2. 导入 IDE（推荐 IntelliJ IDEA） -3. 运行对应 demo 文件即可测试功能（确保模型文件已准备好） +* `face-example`：人脸检测、人脸识别等功能示例 +* `vision-example`：通用视觉检测示例：目标检测、目标分割、图像分类等 +* `ocr-example`：OCR文字识别、车牌识别等功能示例 +* `translate-example`：机器翻译功能示例 +* `speech-example`：语音识别、语音合成功能示例 -## 模型说明 +## 运行方式 + +如果你只想运行某个示例，请按以下方式操作： + +1. 打开 IDEA（或你喜欢的 IDE） +2. 选择 **“Open”**，然后仅导入 `examples` 目录下对应的示例项目，例如： + + ``` + examples/face-example + ``` +3. IDEA 会自动识别并加载依赖。若首次导入，请等待 Maven 下载依赖完成。 +4. 请从我们提供的 百度网盘 中下载模型及其附带文件，并在示例代码中将模型路径修改为您本地的实际路径。 +5. 可通过查看每个 Java 文件顶部的注释了解对应功能，或参考 README 文件中对各 Java 文件功能的说明，运行相应的测试方法进行体验。 -- 本示例项目配合 `smartjavaai` 平台使用，模型加载及使用方式已封装好。 -- 支持 CPU 和 GPU 两种运行模式。 -- 所有模型均可通过 Maven 或本地加载方式接入。 ## 联系与支持 diff --git a/examples/face-example/README.md b/examples/face-example/README.md index cbdc26b..c20ce18 100644 --- a/examples/face-example/README.md +++ b/examples/face-example/README.md @@ -24,52 +24,7 @@ src/main/java/smartai/examples/face/ --- -## 🧩 功能模块说明 -### 1. 人脸属性检测 ([FaceAttributeDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java)) -- **功能**：识别性别、年龄、眼镜佩戴状态、种族等属性。 -- **使用模型**：SeetaFace6 等。 - ---- - -### 2. 表情识别 ([ExpressionRecDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java)) -- **功能**：识别 7 种面部表情：中性、高兴、悲伤、惊讶、恐惧、厌恶、愤怒。 -- **支持模式**：单人、多人、摄像头实时检测。 - ---- - -### 3. 人脸检测 ([FaceDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java)) -- **功能**：识别图像或视频中的人脸区域，并返回人脸边界框。 -- **支持模型**：RetinaFace、SeetaFace6。 ---- - -### 4. 人脸识别 ([FaceRecDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java)) -- **功能**：提取人脸特征、进行人脸比对（1:1）、人脸搜索（1:N）、人脸注册管理。 -- **支持数据库**：SQLite、Milvus 向量数据库。 - ---- - -### 5. 活体检测 ([LivenessDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java)) -- **功能**：判断输入图像中人脸是否为真实人脸（非照片、视频伪造）。 -- **支持模型**：IIC-FL、MiniVision（双模型融合）。 - ---- - -### 6. 人脸质量评估 ([FaceQualityDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java)) -- **功能**：评估人脸图像的质量指标，包括： - - 亮度 (Brightness) - - 完整度 (Completeness) - - 清晰度 (Clarity) - - 姿态 (Pose) - - 分辨率 (Resolution) - ---- - -### 7. 工具类 ([ViewerFrame.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/ViewerFrame.java)) -- **功能**：GUI 显示组件，用于展示图像处理结果（如人脸框、表情、活体状态等）。 -- **用途**：支持摄像头实时检测时的结果可视化。 - ---- ## ⚙️ 配置要求 @@ -83,15 +38,21 @@ src/main/java/smartai/examples/face/ --- -## 🚀 快速开始 -1. 克隆项目到本地： +## 运行方式 -2. 导入项目至 IntelliJ IDEA。 +如果你只想运行某个示例，请按以下方式操作： -3. 根据需要修改模型路径（见各 demo 中注释）。 +1. 打开 IDEA（或你喜欢的 IDE） +2. 选择 **“Open”**，然后仅导入 `examples` 目录下对应的示例项目，例如： + + ``` + examples/face-example + ``` +3. IDEA 会自动识别并加载依赖。若首次导入，请等待 Maven 下载依赖完成。 +4. 请从我们提供的 百度网盘 中下载模型及其附带文件，并在示例代码中将模型路径修改为您本地的实际路径。 +5. 可通过查看每个 Java 文件顶部的注释了解对应功能，或参考 README 文件中对各 Java 文件功能的说明，运行相应的测试方法进行体验。 -4. 运行对应的 JUnit 测试类方法即可体验各项功能。 --- diff --git a/examples/face-example/pom.xml b/examples/face-example/pom.xml index 40ed531..ba03477 100644 --- a/examples/face-example/pom.xml +++ b/examples/face-example/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.22 + 1.0.27 smartai.examples.face.facedet.FaceDetDemo @@ -34,7 +34,7 @@ cn.smartjavaai - smartjavaai-bom + bom ${smartjavaai.version} pom @@ -55,11 +55,7 @@ commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +69,6 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - com.alibaba @@ -94,17 +85,9 @@ cn.smartjavaai - smartjavaai-face - 1.0.22 + face - - cn.smartjavaai - smartjavaai-all - 1.0.22 - - - ai.djl.pytorch @@ -187,6 +170,15 @@ 2.5.1 runtime + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + + + @@ -226,42 +218,7 @@ - - - org.bytedeco - javacpp - ${javacv.version} - ${javacv.platform.linux-arm64} - - - org.bytedeco - ffmpeg - 6.1.1-1.5.10 - ${javacv.platform.linux-arm64} - - - - org.bytedeco - openblas - 0.3.26-1.5.10 - ${javacv.platform.linux-arm64} - - - - org.bytedeco - opencv - 4.9.0-1.5.10 - ${javacv.platform.linux-arm64} - - - - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.5.1 - runtime - @@ -292,18 +249,26 @@ + + + + + + + + + + + + + + - aliyunmaven - 阿里云公共仓库 - https://maven.aliyun.com/repository/public - - true - - - false - + central + https://repo1.maven.org/maven2/ + diff --git a/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java b/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java deleted file mode 100644 index 64c94ff..0000000 --- a/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java +++ /dev/null @@ -1,25 +0,0 @@ -package smartai.examples.face; - -import cn.smartjavaai.common.utils.VideoUtils; -import org.bytedeco.ffmpeg.global.avcodec; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.FFmpegFrameRecorder; - -/** - * 视频预处理 - * @author dwj - * @date 2025/7/17 - */ -public class VideoDemo { - - public static void main(String[] args) { - try { - VideoUtils.rotateVideo("/Users/wenjie/Downloads/girl.mp4", "/Users/wenjie/Downloads/girl_rotate.mp4", 180,"mp4", avcodec.AV_CODEC_ID_H264); - } catch (FFmpegFrameRecorder.Exception e) { - throw new RuntimeException(e); - } catch (FFmpegFrameGrabber.Exception e) { - throw new RuntimeException(e); - } - } - -} diff --git a/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java index 48f0c8e..b019d93 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java @@ -1,10 +1,14 @@ package smartai.examples.face.attribute; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.entity.face.FaceAttribute; import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.face.config.FaceAttributeConfig; import cn.smartjavaai.face.config.FaceDetConfig; import cn.smartjavaai.face.enums.FaceAttributeModelEnum; @@ -16,6 +20,7 @@ import cn.smartjavaai.face.model.facedect.FaceDetModel; import cn.smartjavaai.face.utils.FaceUtils; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; import javax.imageio.ImageIO; @@ -28,23 +33,32 @@ import java.util.List; /** * 人脸属性检测demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j public class FaceAttributeDetDemo { + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + public FaceAttributeModel getFaceAttributeModel() { FaceAttributeConfig config = new FaceAttributeConfig(); config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + config.setModelPath("C:/Users/DengWenJie/Downloads/sf3.0_models/sf3.0_models"); return FaceAttributeModelFactory.getInstance().getModel(config); } public FaceDetModel getFaceDetModel() { //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; + String modelPath = "C:/Users/DengWenJie/Downloads/sf3.0_models/sf3.0_models"; FaceDetConfig faceDetectModelConfig = new FaceDetConfig(); faceDetectModelConfig.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); faceDetectModelConfig.setModelPath(modelPath); @@ -59,10 +73,12 @@ public class FaceAttributeDetDemo { public void testFaceAttributeDetect(){ try { FaceAttributeModel faceAttributeModel = getFaceAttributeModel(); - DetectionResponse detectionResponse = faceAttributeModel.detect("src/main/resources/iu_1.jpg"); + ////创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + DetectionResponse detectionResponse = faceAttributeModel.detect(image); //绘制并导出人脸属性图片，小人脸仅有人脸框 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); - FaceUtils.drawBoxesWithFaceAttribute(image, detectionResponse,"C:/Users/Administrator/Downloads/double_person_.png"); + BufferedImage bufferedImage = ImageUtils.toBufferedImage(image); + FaceUtils.drawBoxesWithFaceAttribute(bufferedImage, detectionResponse,"C:/Users/Administrator/Downloads/double_person_.png"); log.info("人脸属性检测结果：{}", JSONObject.toJSONString(detectionResponse)); } catch (Exception e) { e.printStackTrace(); @@ -76,30 +92,15 @@ public class FaceAttributeDetDemo { public void testFaceAttributeDetect2(){ try { FaceAttributeModel faceAttributeModel = getFaceAttributeModel(); - FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/iu_1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + FaceAttribute faceAttribute = faceAttributeModel.detectTopFace(image); log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); } catch (Exception e) { e.printStackTrace(); } } - /** - * 图片多人脸属性检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testFaceAttributeDetect3(){ - try { - FaceAttributeModel faceAttributeModel = getFaceAttributeModel(); - FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/iu_1.jpg"); - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); - } catch (Exception e) { - e.printStackTrace(); - } - //人脸检测 - - - - } /** * 图片单人脸人脸属性检测（基于已检测出的人脸区域和关键点） @@ -109,8 +110,8 @@ public class FaceAttributeDetDemo { try { FaceDetModel faceDetModel = getFaceDetModel(); FaceAttributeModel faceAttributeModel = getFaceAttributeModel(); - //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); diff --git a/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java index a187ea9..00435a4 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java @@ -2,6 +2,8 @@ package smartai.examples.face.expression; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; @@ -10,6 +12,7 @@ import cn.smartjavaai.common.entity.face.ExpressionResult; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.enums.face.FacialExpression; import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.utils.BufferedImageUtils; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.common.utils.OpenCVUtils; import cn.smartjavaai.face.config.FaceDetConfig; @@ -26,6 +29,7 @@ import cn.smartjavaai.face.model.liveness.LivenessDetModel; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import nu.pattern.OpenCV; +import org.junit.BeforeClass; import org.junit.Test; import org.opencv.core.Mat; import org.opencv.core.Size; @@ -46,6 +50,8 @@ import java.util.List; /** * 表情识别demo * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -54,16 +60,30 @@ public class ExpressionRecDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** - * 获取人脸检测模型 + * 获取人脸检测模型（均衡模型） + * 均衡模型：兼顾速度和精度 + * 注意事项：SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath * @return */ public FaceDetModel getFaceDetModel(){ FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 - config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 - config.setDevice(device); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.MTCNN); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/mtcnn"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return FaceDetModelFactory.getInstance().getModel(config); } @@ -75,7 +95,7 @@ public class ExpressionRecDemo { public ExpressionModel getExpressionModel(){ FaceExpressionConfig config = new FaceExpressionConfig(); config.setModelEnum(ExpressionModelEnum.FrEmotion); - config.setModelPath("/Users/xxx/Documents/develop/model/emotion/fr_expression.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/emotion/fr_expression.onnx"); config.setDevice(device); config.setAlign(true); config.setDetectModel(getFaceDetModel()); @@ -90,7 +110,9 @@ public class ExpressionRecDemo { public void testExpressionDetect() { try { ExpressionModel model = getExpressionModel(); - R result = model.detectTopFace("src/main/resources/emotion/happy.png"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/emotion/happy.png"); + R result = model.detectTopFace(image); if(result.isSuccess()){ log.info("识别结果：{}", JSONObject.toJSONString(result.getData().getExpression().getDescription())); }else{ @@ -109,7 +131,9 @@ public class ExpressionRecDemo { public void testExpressionDetect2() { try { ExpressionModel model = getExpressionModel(); - R result = model.detect("src/main/resources/emotion/happy.png"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/emotion/happy.png"); + R result = model.detect(image); if(result.isSuccess()){ //log.info("识别结果：{}", JSONObject.toJSONString(result.getData())); for (DetectionInfo detectionInfo : result.getData().getDetectionInfoList()) { @@ -133,7 +157,8 @@ public class ExpressionRecDemo { try { FaceDetModel faceDetModel = getFaceDetModel(); ExpressionModel model = getExpressionModel(); - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/emotion/happy.png").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/emotion/happy.png"); R detResult = faceDetModel.detect(image); if(detResult.isSuccess()){ R> result = model.detect(image, detResult.getData()); @@ -162,7 +187,8 @@ public class ExpressionRecDemo { try { FaceDetModel faceDetModel = getFaceDetModel(); ExpressionModel model = getExpressionModel(); - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/emotion/happy.png").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/emotion/happy.png"); R detResult = faceDetModel.detect(image); if(detResult.isSuccess()){ for (DetectionInfo detectionInfo : detResult.getData().getDetectionInfoList()) { @@ -188,15 +214,16 @@ public class ExpressionRecDemo { public void testExpressionDetectAndDraw(){ try { ExpressionModel model = getExpressionModel(); - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/emotion/surprise.png").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/emotion/surprise.png"); R result = model.detect(image); if(result.isSuccess()){ //log.info("识别结果：{}", JSONObject.toJSONString(result.getData())); for (DetectionInfo detectionInfo : result.getData().getDetectionInfoList()) { log.info("识别结果：{}", JSONObject.toJSONString(detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription())); - ImageUtils.drawImageRectWithText(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription(), Color.red); + ImageUtils.drawRectAndText(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription()); } - ImageUtils.saveImage(image, "output/detect.jpg"); + ImageUtils.save(image, "output/detect.jpg"); }else{ log.info("识别失败：{}", result.getMessage()); } @@ -209,7 +236,7 @@ public class ExpressionRecDemo { * 摄像头表情识别 * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 */ - @Test +// @Test public void testExpressionDetectCamera(){ try { ExpressionModel expressionModel = getExpressionModel(); @@ -248,7 +275,7 @@ public class ExpressionRecDemo { JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); } ViewerFrame frame = new ViewerFrame(width, height); - ImageFactory factory = ImageFactory.getInstance(); + SmartImageFactory factory = SmartImageFactory.getInstance(); Size size = new Size(width, height); while (capture.isOpened()) { @@ -257,19 +284,18 @@ public class ExpressionRecDemo { } Mat resizeImage = new Mat(); Imgproc.resize(image, resizeImage, size); - Image img = factory.fromImage(resizeImage); - BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); - R detectedResult = expressionModel.detect(bufferedImage); + Image img = factory.fromMat(resizeImage); + R detectedResult = expressionModel.detect(img); if(!detectedResult.isSuccess()){ log.debug("识别失败：{}", detectedResult.getMessage()); continue; } for(DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){ DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); - String text = detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription() + ":" + detectionInfo.getFaceInfo().getExpressionResult().getScore(); - ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.red); + String text = detectionInfo.getFaceInfo().getExpressionResult().getExpression().getLabel() + ":" + detectionInfo.getFaceInfo().getExpressionResult().getScore(); + ImageUtils.drawRectAndText(img, detectionRectangle, text); } - frame.showImage(bufferedImage); + frame.showImage(ImageUtils.toBufferedImage(img)); } capture.release(); diff --git a/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java index 2e084a0..2593894 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java @@ -2,6 +2,9 @@ package smartai.examples.face.facedet; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; +import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; @@ -16,9 +19,11 @@ import cn.smartjavaai.face.enums.FaceDetModelEnum; import cn.smartjavaai.face.factory.FaceDetModelFactory; import cn.smartjavaai.face.model.facedect.FaceDetModel; import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import cn.smartjavaai.face.utils.FaceUtils; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import nu.pattern.OpenCV; +import org.junit.BeforeClass; import org.junit.Test; import org.opencv.core.Mat; import org.opencv.core.Size; @@ -32,13 +37,13 @@ import javax.swing.*; import java.awt.*; import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; /** * 人脸检测模型demo - * 支持系统：windows 64位，linux 64位, macos M系列 - * 支持功能：人脸检测 * 模型下载地址：https://pan.baidu.com/s/1d2YlJ2YOdGn3Y-AegyAhmQ?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -47,17 +52,68 @@ public class FaceDetDemo { public static String imgPath = "src/main/resources/iu_1.jpg"; + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** - * 获取人脸检测模型 - * 注意事项：高精度模型，速度较慢 + * 获取人脸检测模型（均衡模型） + * 均衡模型：兼顾速度和精度 + * 注意事项：SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath * @return */ public FaceDetModel getFaceDetModel(){ FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 - config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.MTCNN); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/mtcnn"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); + return FaceDetModelFactory.getInstance().getModel(config); + } + + + /** + * 获取人脸检测模型（高精度模型） + * 注意事项：高精度模型，识别准确度高，速度慢 + * @return + */ + public FaceDetModel getProFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.RETINA_FACE); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/face_model/retinaface.pt"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); + return FaceDetModelFactory.getInstance().getModel(config); + } + + /** + * 获取人脸检测模型（极速模型） + * 注意事项：极速模型，识别准确度低，速度快 + * @return + */ + public FaceDetModel getFastFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.YOLOV5_FACE_320); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/yolo-face/yolov5face-n-0.5-320x320.onnx"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return FaceDetModelFactory.getInstance().getModel(config); } @@ -70,42 +126,32 @@ public class FaceDetDemo { FaceDetConfig config = new FaceDetConfig(); //指定模型 config.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); - //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） + //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234） config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + config.setConfidenceThreshold(0.9); return FaceDetModelFactory.getInstance().getModel(config); } + /** - * 人脸检测(默认配置) - * 使用默认模型参数检测，默认模型：retinaface，需联网，会自动下载模型 - * 图片参数：图片路径 + * 人脸检测 + * 注意事项： + * 1、此用例使用均衡模型，可以切换高精度模型或极速模型 */ @Test public void testFaceDetect(){ - try { - FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel(); - R detectedResult = faceModel.detect(imgPath); - if(detectedResult.isSuccess()){ - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); - }else{ - log.info("人脸检测失败：{}", detectedResult.getMessage()); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测(自定义模型参数) - * 图片参数：图片路径 - */ - @Test - public void testFaceDetectCustomConfig(){ try { FaceDetModel faceModel = getFaceDetModel(); - R detectedResult = faceModel.detect(imgPath); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(imgPath); + R detectedResult = faceModel.detect(image); if(detectedResult.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + //裁剪人脸保存 + for (DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()) { + Image faceImage = FaceUtils.cropFace(image, detectionInfo.getDetectionRectangle()); + ImageUtils.save(faceImage, "output/face_" + detectionInfo.getDetectionRectangle().getX() + "_" + detectionInfo.getDetectionRectangle().getY() + ".jpg"); + } }else{ log.info("人脸检测失败：{}", detectedResult.getMessage()); } @@ -122,7 +168,12 @@ public class FaceDetDemo { public void testFaceDetectAndDraw(){ try { FaceDetModel faceModel = getFaceDetModel(); - faceModel.detectAndDraw("src/main/resources/largest_selfie.jpg","output/largest_selfie_detected.png"); + R detectedResult = faceModel.detectAndDraw("src/main/resources/largest_selfie.jpg","output/largest_selfie_detected.png"); + if(detectedResult.isSuccess()){ + log.info("人脸检测成功：{}", JsonUtils.toJson(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -136,15 +187,14 @@ public class FaceDetDemo { public void testFaceDetectAndDraw2(){ try { FaceDetModel faceModel = getFaceDetModel(); - BufferedImage image = null; - String imagePath = "src/main/resources/largest_selfie.jpg"; - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - R detectedImage = faceModel.detectAndDraw(image); - if(detectedImage.isSuccess()){ - log.info("人脸检测成功"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(imgPath); + R detectionResponseR = faceModel.detectAndDraw(image); + if(detectionResponseR.isSuccess()){ + log.info("人脸检测成功：{}", JsonUtils.toJson(detectionResponseR.getData())); + ImageUtils.save(detectionResponseR.getData().getDrawnImage(), "output/iu_1_detect.png"); }else{ - log.info("人脸检测失败：{}", detectedImage.getMessage()); + log.info("人脸检测失败：{}", detectionResponseR.getMessage()); } } catch (Exception e) { throw new RuntimeException(e); @@ -153,58 +203,18 @@ public class FaceDetDemo { } - /** - * 人脸检测（离线模型） - */ - @Test - public void testDetectFaceOffine(){ - try { - FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸模型 - //模型路径,不同模型下载路径请参看文档 - config.setModelPath("/Users/xxx/Documents/develop/face_model/retinaface.pt"); - FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel(config); - R detectedResult = faceModel.detect(imgPath); - if(detectedResult.isSuccess()){ - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); - }else{ - log.info("人脸检测失败：{}", detectedResult.getMessage()); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测（GPU模式） - */ - @Test - public void testDetectFaceGPU(){ - try { - FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸模型 - config.setDevice(DeviceEnum.GPU); - FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel(config); - R detectedResult = faceModel.detect(imgPath); - if(detectedResult.isSuccess()){ - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); - }else{ - log.info("人脸检测失败：{}", detectedResult.getMessage()); - } - } catch (Exception e) { - e.printStackTrace(); - } - } /** * 人脸检测(Seetaface6) - * 图片参数：图片路径 + * 注意事项：不支持macos */ @Test public void testFaceDetectSeetaface6(){ try { FaceDetModel faceModel = getSeetaface6DetModel(); - R detectedResult = faceModel.detect(imgPath); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(imgPath); + R detectedResult = faceModel.detect(image); if(detectedResult.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); }else{ @@ -218,12 +228,12 @@ public class FaceDetDemo { /** * 摄像头人脸检测 - * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 + * 注意事项：实时检测，需要使用极速模型 */ @Test public void testDetectCamera(){ try { - FaceDetModel faceModel = getFaceDetModel(); + FaceDetModel faceModel = getFastFaceDetModel(); OpenCV.loadShared(); VideoCapture capture = new VideoCapture(0); if (!capture.isOpened()) { @@ -259,7 +269,7 @@ public class FaceDetDemo { JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); } ViewerFrame frame = new ViewerFrame(width, height); - ImageFactory factory = ImageFactory.getInstance(); + SmartImageFactory factory = SmartImageFactory.getInstance(); Size size = new Size(width, height); while (capture.isOpened()) { @@ -268,9 +278,8 @@ public class FaceDetDemo { } Mat resizeImage = new Mat(); Imgproc.resize(image, resizeImage, size); - Image img = factory.fromImage(resizeImage); - BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); - R detectedResult = faceModel.detect(bufferedImage); + Image img = factory.fromMat(resizeImage); + R detectedResult = faceModel.detect(img); if(!detectedResult.isSuccess()){ log.debug("识别失败：{}", detectedResult.getMessage()); continue; @@ -281,11 +290,10 @@ public class FaceDetDemo { if(detectionInfo.getScore() > 0){ text = detectionInfo.getScore() + ""; } - ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.red); + ImageUtils.drawRectAndText(img, detectionRectangle, text); } - frame.showImage(bufferedImage); + frame.showImage(ImageUtils.toBufferedImage(img)); } - capture.release(); System.exit(0); } catch (Exception e) { diff --git a/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java index 3cee699..9cb1a1f 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java @@ -1,9 +1,14 @@ package smartai.examples.face.facerec; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.SimilarityType; +import cn.smartjavaai.common.utils.BufferedImageUtils; +import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.face.config.FaceDetConfig; import cn.smartjavaai.face.config.FaceRecConfig; import cn.smartjavaai.face.constant.FaceDetectConstant; @@ -12,29 +17,28 @@ import cn.smartjavaai.face.entity.FaceSearchParams; import cn.smartjavaai.face.enums.FaceDetModelEnum; import cn.smartjavaai.face.enums.FaceRecModelEnum; import cn.smartjavaai.face.enums.IdStrategy; -import cn.smartjavaai.face.enums.SimilarityType; import cn.smartjavaai.face.factory.FaceDetModelFactory; import cn.smartjavaai.face.factory.FaceRecModelFactory; import cn.smartjavaai.face.model.facedect.FaceDetModel; import cn.smartjavaai.face.model.facerec.FaceRecModel; -import cn.smartjavaai.face.utils.SimilarityUtil; import cn.smartjavaai.face.vector.config.MilvusConfig; import cn.smartjavaai.face.vector.config.SQLiteConfig; import cn.smartjavaai.face.vector.entity.FaceVector; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; +import java.awt.image.BufferedImage; +import java.io.IOException; import java.util.List; /** - * FaceNet人脸算法模型demo - * 支持系统：windows 64位，linux 64位，macOS M系列芯片 - * 支持功能：人脸特征提取、人脸比对（1：1）、人脸比对（1：N）、人脸注册 + * 人脸识别模型demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/4/11 */ @Slf4j public class FaceRecDemo { @@ -42,29 +46,87 @@ public class FaceRecDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** - * 获取人脸检测模型 + * 获取人脸检测模型（均衡模型） + * 均衡模型：兼顾速度和精度 + * 注意事项：SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath * @return */ public FaceDetModel getFaceDetModel(){ FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 - config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 - config.setDevice(device); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.MTCNN); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/mtcnn"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return FaceDetModelFactory.getInstance().getModel(config); } /** - * 获取人脸识别模型 + * 获取人脸检测模型（高精度模型） + * 注意事项： + * 1、高精度模型，识别准确度高，速度慢 + * 2、具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html + * @return + */ + public FaceDetModel getProFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.RETINA_FACE); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/face_model/retinaface.pt"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); + return FaceDetModelFactory.getInstance().getModel(config); + } + + /** + * 获取人脸检测模型（极速模型） + * 注意事项： + * 1、极速模型，识别准确度低，速度快 + * 2、具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html + * @return + */ + public FaceDetModel getFastFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.YOLOV5_FACE_320); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/face_model/yolo-face/yolov5face-n-0.5-320x320.onnx"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); + return FaceDetModelFactory.getInstance().getModel(config); + } + + + /** + * 获取人脸识别模型（高精度，速度慢） + * 追求准确度可以使用 + * 也可以使用其他模型，具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html * @return */ public FaceRecModel getFaceRecModel(){ FaceRecConfig config = new FaceRecConfig(); - config.setModelEnum(FaceRecModelEnum.FACENET_MODEL); -// config.setModelPath("/Users/xxx/Documents/develop/model/elasticface.pt"); -// config.setModelPath("/Users/xxx/Documents/develop/model/InsightFace/model_mobilefacenet.pt"); + //高精度模型，速度慢 + config.setModelEnum(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL); + //模型路径，请下载模型并替换为本地路径：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/recognition/InsightFace/model_ir_se50.pt"); //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false config.setCropFace(true); //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 @@ -75,19 +137,43 @@ public class FaceRecDemo { return FaceRecModelFactory.getInstance().getModel(config); } + /** + * 获取人脸识别模型（高速模型，精度一般） + * 追求速度可以使用 + * 也可以使用其他模型，具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html + * @return + */ + public FaceRecModel getHighSpeedFaceRecModel(){ + FaceRecConfig config = new FaceRecConfig(); + //模型枚举 + config.setModelEnum(FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL); + //模型路径，请下载模型并替换为本地路径：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/recognition/InsightFace/model_mobilefacenet.pt"); + //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false + config.setCropFace(true); + //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 + config.setAlign(false); + config.setDevice(device); + //指定人脸检测模型 + config.setDetectModel(getFastFaceDetModel()); + return FaceRecModelFactory.getInstance().getModel(config); + } + /** * 获取人脸识别模型(带向量数据库配置) * @return */ public FaceRecModel getFaceRecModelWithDbConfig(){ FaceRecConfig config = new FaceRecConfig(); - config.setModelEnum(FaceRecModelEnum.ELASTIC_FACE_MODEL);//人脸检测模型 - config.setModelPath("/Users/xxx/Documents/develop/model/elasticface.pt"); + //高精度模型，速度慢,追求速度请更换高速模型，具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html + config.setModelEnum(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL); + //模型路径，请下载模型并替换为本地路径：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/recognition/InsightFace/model_ir_se50.pt"); //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false config.setCropFace(true); //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 config.setAlign(true); - //指定人脸检测模型 + //指定人脸检测模型，可切换人脸检测模型（极速：getFastFaceDetModel，高精度：getProFaceDetModel），具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html config.setDetectModel(getFaceDetModel()); config.setDevice(device); @@ -95,9 +181,9 @@ public class FaceRecDemo { MilvusConfig vectorDBConfig = new MilvusConfig(); vectorDBConfig.setHost("127.0.0.1"); vectorDBConfig.setPort(19530); - //vectorDBConfig.setUsername("root"); - //vectorDBConfig.setPassword("Milvus"); - //vectorDBConfig.setCollectionName("face5"); +// vectorDBConfig.setUsername("root"); +// vectorDBConfig.setPassword("Milvus"); +// vectorDBConfig.setCollectionName("face6"); //ID策略：自动生成 vectorDBConfig.setIdStrategy(IdStrategy.AUTO); //索引类型:内积 (Inner Product) 不建议修改 @@ -112,12 +198,15 @@ public class FaceRecDemo { */ public FaceRecModel getFaceRecModelWithSQLiteConfig(){ FaceRecConfig config = new FaceRecConfig(); - config.setModelEnum(FaceRecModelEnum.FACENET_MODEL);//人脸检测模型 + //高精度模型，速度慢, 追求速度请更换高速模型，具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html + config.setModelEnum(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL); + //模型路径，请下载模型并替换为本地路径：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/recognition/InsightFace/model_ir_se50.pt"); //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false config.setCropFace(true); //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 config.setAlign(true); - //指定人脸检测模型 + //指定人脸检测模型，可切换人脸检测模型（极速：getFastFaceDetModel，高精度：getProFaceDetModel），具体其他模型参数可以查看文档：http://doc.smartjavaai.cn/face.html config.setDetectModel(getFaceDetModel()); config.setDevice(device); @@ -138,9 +227,12 @@ public class FaceRecDemo { @Test public void testExtractFeatures(){ try { + //高精度模型，速度慢, 追求速度请更换高速模型: getHighSpeedFaceRecModel FaceRecModel faceRecModel = getFaceRecModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //提取图片中所有人脸特征 - R faceResult = faceRecModel.extractFeatures("src/main/resources/iu_1.jpg"); + R faceResult = faceRecModel.extractFeatures(image); if(faceResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(faceResult.getData())); }else{ @@ -162,15 +254,60 @@ public class FaceRecDemo { @Test public void featureComparison(){ try { + //高精度模型，速度慢, 追求速度请更换高速模型: getHighSpeedFaceRecModel FaceRecModel faceRecModel = getFaceRecModel(); //基于图像直接比对人脸特征 R similarResult = faceRecModel.featureComparison("src/main/resources/iu_1.jpg","src/main/resources/iu_2.jpg"); if(similarResult.isSuccess()){ //相似度阈值不同模型不同，具体参看文档 log.info("人脸比对相似度：{}", JSONObject.toJSONString(similarResult.getData())); + //不同模型的相似度标准不同。当前阈值仅适用于 insight_face 模型，切换模型时请相应调整阈值，详情请参考文档。 + if(similarResult.getData() >= 0.62f){ + log.info("识别为同一人"); + }else{ + log.info("识别为不同人"); + } }else{ log.info("人脸比对失败：{}", similarResult.getMessage()); } + + } + catch (Exception e){ + e.printStackTrace(); + } + } + + /** + * 人脸比对1：1（基于图像直接比对） + * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 比对两张图片中提取的人脸特征。（接口内自动完成） + * 注意事项： + * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 + * 2、若人脸朝向不正，可开启人脸对齐以提升特征提取准确度。（方法参考自定义配置人脸特征提取） + * @throws Exception + */ + @Test + public void featureComparison3(){ + try { + //高精度模型，速度慢, 追求速度请更换高速模型: getHighSpeedFaceRecModel + FaceRecModel faceRecModel = getFaceRecModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image1 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + Image image2 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_2.jpg"); + //基于图像直接比对人脸特征 + R similarResult = faceRecModel.featureComparison(image1, image2); + if(similarResult.isSuccess()){ + //相似度阈值不同模型不同，具体参看文档 + log.info("人脸比对相似度：{}", JSONObject.toJSONString(similarResult.getData())); + //不同模型的相似度标准不同。当前阈值仅适用于 insight_face 模型，切换模型时请相应调整阈值，详情请参考文档。 + if(similarResult.getData() >= 0.62f){ + log.info("识别为同一人"); + }else{ + log.info("识别为不同人"); + } + }else{ + log.info("人脸比对失败：{}", similarResult.getMessage()); + } + } catch (Exception e){ e.printStackTrace(); @@ -188,9 +325,12 @@ public class FaceRecDemo { @Test public void featureComparison2(){ try { + //高精度模型，速度慢, 追求速度请更换高速模型: getHighSpeedFaceRecModel FaceRecModel faceRecModel = getFaceRecModel(); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult1 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image1 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + R featureResult1 = faceRecModel.extractTopFaceFeature(image1); if(featureResult1.isSuccess()){ log.info("图片1人脸特征提取成功：{}", JSONObject.toJSONString(featureResult1.getData())); }else{ @@ -198,7 +338,8 @@ public class FaceRecDemo { return; } //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_2.jpg"); + Image image2 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_2.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature(image2); if(featureResult2.isSuccess()){ log.info("图片2人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -208,6 +349,12 @@ public class FaceRecDemo { //计算相似度 float similar = faceRecModel.calculSimilar(featureResult1.getData(), featureResult2.getData()); log.info("相似度：{}", similar); + //不同模型的相似度标准不同。当前阈值仅适用于 insight_face 模型，切换模型时请相应调整阈值，详情请参考文档。 + if(similar >= 0.62f){ + log.info("识别为同一人"); + }else{ + log.info("识别为不同人"); + } } catch (Exception e){ e.printStackTrace(); @@ -226,14 +373,17 @@ public class FaceRecDemo { @Test public void searchFace(){ try { + //高精度模型，速度慢, 追求速度请更换高速模型 FaceRecModel faceRecModel = getFaceRecModelWithDbConfig(); //等待加载人脸库结束 while (!faceRecModel.isLoadFaceCompleted()){ Thread.sleep(100); } log.info("====================人脸注册=========================="); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); + R featureResult = faceRecModel.extractTopFaceFeature(image); if(featureResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); }else{ @@ -255,21 +405,23 @@ public class FaceRecDemo { }else{ log.info("注册失败：{}", registerResult.getMessage()); } - /*log.info("====================人脸更新=========================="); + log.info("====================人脸更新=========================="); //更新人脸 只支持自定义ID：vectorDBConfig.setIdStrategy(IdStrategy.CUSTOM); - FaceRegisterInfo updateInfo = new FaceRegisterInfo(); - //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 - JSONObject metadataJsonUpdate = new JSONObject(); - metadataJsonUpdate.put("name", "iu_update"); - metadataJsonUpdate.put("age", "25"); - updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); - //更新必须设置ID,只有 - updateInfo.setId(registerResult.getData()); - faceRecModel.upsertFace(updateInfo, "src/main/resources/iu_2.jpg"); - log.info("更新人脸成功");*/ +// FaceRegisterInfo updateInfo = new FaceRegisterInfo(); +// //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 +// JSONObject metadataJsonUpdate = new JSONObject(); +// metadataJsonUpdate.put("name", "iu_update"); +// metadataJsonUpdate.put("age", "25"); +// updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); +// //更新必须设置ID,只有 +// updateInfo.setId(registerResult.getData()); +// Image image2 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_2.jpg"); +// faceRecModel.upsertFace(updateInfo, image2); +// log.info("更新人脸成功"); log.info("====================人脸查询=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_3.jpg"); + Image image3 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_3.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature(image3); if(featureResult2.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -278,8 +430,7 @@ public class FaceRecDemo { } FaceSearchParams faceSearchParams = new FaceSearchParams(); faceSearchParams.setTopK(1); - faceSearchParams.setThreshold(0.8f); - +// faceSearchParams.setThreshold(0.62f); List faceSearchResults = faceRecModel.search(featureResult2.getData(), faceSearchParams); // R faceSearchResults = faceModel.search("src/main/resources/face/iu_3.jpg", faceSearchParams); log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); @@ -303,6 +454,7 @@ public class FaceRecDemo { @Test public void searchFace2(){ try { + //高精度模型，速度慢, 追求速度请更换高速模型 FaceRecModel faceRecModel = getFaceRecModelWithSQLiteConfig(); //等待加载人脸库结束 while (!faceRecModel.isLoadFaceCompleted()){ @@ -310,7 +462,9 @@ public class FaceRecDemo { } log.info("====================人脸注册=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + R featureResult = faceRecModel.extractTopFaceFeature(image); if(featureResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); }else{ @@ -342,11 +496,13 @@ public class FaceRecDemo { updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); //更新必须设置ID,只有 updateInfo.setId(registerResult.getData()); - faceRecModel.upsertFace(updateInfo, "src/main/resources/iu_2.jpg"); + Image image2 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_2.jpg"); + faceRecModel.upsertFace(updateInfo, image2); log.info("更新人脸成功"); log.info("====================人脸查询=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_3.jpg"); + Image image3 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_3.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature(image3); if(featureResult2.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -355,7 +511,7 @@ public class FaceRecDemo { } FaceSearchParams faceSearchParams = new FaceSearchParams(); faceSearchParams.setTopK(1); - faceSearchParams.setThreshold(0.8f); + //faceSearchParams.setThreshold(0.62f); List faceSearchResults = faceRecModel.search(featureResult2.getData(), faceSearchParams); log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); log.info("====================人脸删除=========================="); @@ -367,6 +523,57 @@ public class FaceRecDemo { } } + /** + * 人脸查询及绘制 + * + * @throws Exception + */ + @Test + public void searchFace3(){ + try { + //高精度模型，速度慢, 追求速度请更换高速模型 + FaceRecModel faceRecModel = getFaceRecModelWithSQLiteConfig(); + //等待加载人脸库结束 + while (!faceRecModel.isLoadFaceCompleted()){ + Thread.sleep(100); + } + log.info("====================人脸注册=========================="); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); + //人脸注册信息 + FaceRegisterInfo faceRegisterInfo = new FaceRegisterInfo(); + //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 + JSONObject metadataJson = new JSONObject(); + metadataJson.put("name", "iu"); + metadataJson.put("age", "25"); + faceRegisterInfo.setMetadata(metadataJson.toJSONString()); + //可自定义 ID，若未设置则自动生成。 + //faceRegisterInfo.setId("00001"); + //人脸注册，返回人脸库ID + R registerResult = faceRecModel.register(faceRegisterInfo, image); + if(registerResult.isSuccess()){ + log.info("注册成功：ID-{}", registerResult.getData()); + }else{ + log.info("注册失败：{}", registerResult.getMessage()); + } + log.info("====================人脸查询=========================="); + //特征提取（提取分数最高人脸特征）,适用于单人脸场景 + Image image3 = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_3.jpg"); + FaceSearchParams faceSearchParams = new FaceSearchParams(); + faceSearchParams.setTopK(1); + //faceSearchParams.setThreshold(0.62f); + //图片中只会显示Metadata信息中name的字段 + Image drawSearchResult = faceRecModel.drawSearchResult(image3, faceSearchParams, "name"); + ImageUtils.save(drawSearchResult, "output/search_result.jpg"); + log.info("====================人脸删除=========================="); + faceRecModel.removeRegister(registerResult.getData()); + log.info("人脸删除成功"); + } + catch (Exception e){ + e.printStackTrace(); + } + } + /** * 获取人脸信息 @@ -399,7 +606,7 @@ public class FaceRecDemo { public void listFaces(){ //使用ID获取人脸信息 try { - FaceRecModel faceRecModel = getFaceRecModelWithDbConfig(); + FaceRecModel faceRecModel = getFaceRecModelWithSQLiteConfig(); //等待加载人脸库结束 while (!faceRecModel.isLoadFaceCompleted()){ Thread.sleep(100); @@ -419,4 +626,7 @@ public class FaceRecDemo { + + + } diff --git a/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java index b55ea7e..e6854e2 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java @@ -3,6 +3,8 @@ package smartai.examples.face.liveness; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; import cn.hutool.core.lang.UUID; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; @@ -32,6 +34,7 @@ import nu.pattern.OpenCV; import org.bytedeco.javacv.FFmpegFrameGrabber; import org.bytedeco.javacv.Frame; import org.bytedeco.javacv.Java2DFrameUtils; +import org.junit.BeforeClass; import org.junit.Test; import org.opencv.core.Mat; import org.opencv.core.Size; @@ -52,8 +55,8 @@ import java.util.List; /** * 静态活体检测demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/1 */ @Slf4j public class LivenessDetDemo { @@ -61,6 +64,14 @@ public class LivenessDetDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** @@ -72,7 +83,7 @@ public class LivenessDetDemo { config.setModelEnum(LivenessModelEnum.IIC_FL_MODEL); config.setDevice(device); //需替换为实际模型存储路径 - config.setModelPath("/Users/xxx/Documents/develop/model/anti/IIC_Fl.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/anti/IIC_Fl.onnx"); //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); /*视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 @@ -96,9 +107,9 @@ public class LivenessDetDemo { config.setModelEnum(LivenessModelEnum.MINI_VISION_MODEL); config.setDevice(device); //模型1路径：需替换为实际模型存储路径 - config.setModelPath("/Users/xxx/Documents/develop/model/live/2.7_80x80_MiniFASNetV2.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/live/2.7_80x80_MiniFASNetV2.onnx"); //SE模型路径：需替换为实际模型存储路径 - config.putCustomParam("seModelPath", "/Users/xxx/Documents/develop/model/live/4_0_0_80x80_MiniFASNetV1SE.onnx"); + config.putCustomParam("seModelPath", "/Users/wenjie/Documents/develop/model/live/4_0_0_80x80_MiniFASNetV1SE.onnx"); //人脸活体阈值,可选,超过阈值则认为是真人，低于阈值是非活体 config.setRealityThreshold(0.5f); /*视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 @@ -114,15 +125,21 @@ public class LivenessDetDemo { /** - * 获取人脸检测模型 + * 获取人脸检测模型（均衡模型） + * 均衡模型：兼顾速度和精度 + * 注意事项：SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath * @return */ public FaceDetModel getFaceDetModel(){ FaceDetConfig config = new FaceDetConfig(); - config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 - config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 - config.setDevice(device); + //人脸检测模型，SmartJavaAI提供了多种模型选择(更多模型，请查看文档)，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(FaceDetModelEnum.MTCNN); + //下载模型并替换本地路径，下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + config.setModelPath("/Users/wenjie/Documents/develop/model/face_model/mtcnn"); + //只返回相似度大于该值的人脸,需要根据实际情况调整，分值越大越严格容易漏检，分值越小越宽松容易误识别 + config.setConfidenceThreshold(0.5f); + //用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return FaceDetModelFactory.getInstance().getModel(config); } @@ -135,10 +152,12 @@ public class LivenessDetDemo { public void testLivenessDetect(){ try { LivenessDetModel livenessDetModel = getLivenessDetModel(); - R response = livenessDetModel.detect("src/main/resources/liveness/1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/liveness/1.jpg"); + R response = livenessDetModel.detect(image); if(response.isSuccess()){ for (DetectionInfo detectionInfo : response.getData().getDetectionInfoList()){ - log.info("活体检测结果：{}", JSONObject.toJSONString(detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription())); + log.info("活体检测结果：{}", JSONObject.toJSONString(detectionInfo)); } }else{ log.info("活体检测失败：{}", response.getMessage()); @@ -155,18 +174,18 @@ public class LivenessDetDemo { public void testLivenessDetectAndDraw(){ try { LivenessDetModel livenessDetModel = getLivenessDetModel(); - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/liveness/1.jpg").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/liveness/1.jpg"); R response = livenessDetModel.detect(image); if(response.isSuccess()){ for (DetectionInfo detectionInfo : response.getData().getDetectionInfoList()){ log.info("活体检测结果：{}", JSONObject.toJSONString(detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription())); - Color color = detectionInfo.getFaceInfo().getLivenessStatus().getStatus() == LivenessStatus.LIVE ? Color.GREEN : Color.RED; - ImageUtils.drawImageRectWithText(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription(), color); + ImageUtils.drawRectAndText(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getLivenessStatus().getStatus().toString()); + ImageUtils.save(image, "output/detect.jpg"); } }else{ log.info("活体检测失败：{}", response.getMessage()); } - ImageUtils.saveImage(image, "output/detect.jpg"); } catch (Exception e) { throw new RuntimeException(e); } @@ -180,10 +199,12 @@ public class LivenessDetDemo { try { LivenessDetModel livenessDetModel = getLivenessDetModel(); //指定文件夹路径 - File dir = new File("face-example/src/main/resources/liveness"); + File dir = new File("src/main/resources/liveness"); File[] files = dir.listFiles(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + SmartImageFactory imageFactory = SmartImageFactory.getInstance(); for (File file : files) { - R response = livenessDetModel.detectTopFace(ImageIO.read(file)); + R response = livenessDetModel.detectTopFace(imageFactory.fromFile(file)); if(response.isSuccess()){ log.info("{}活体检测结果：{},分数：{}", file.getName(), response.getData().getStatus().getDescription(), response.getData().getScore()); }else{ @@ -204,8 +225,8 @@ public class LivenessDetDemo { try { FaceDetModel faceDetectModel = getFaceDetModel(); LivenessDetModel livenessDetModel = getLivenessDetModel(); - // 将图片路径转换为 BufferedImage - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/liveness/1.jpg").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/liveness/1.jpg"); //人脸检测 R detectionResponse = faceDetectModel.detect(image); if(detectionResponse.isSuccess()){ @@ -237,8 +258,8 @@ public class LivenessDetDemo { try { FaceDetModel faceDetModel = getFaceDetModel(); LivenessDetModel livenessDetModel = getMiniVisionLivenessDetModel(); - // 将图片路径转换为 BufferedImage - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/liveness/1.jpg").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/liveness/1.jpg"); R detResult = faceDetModel.detect(image); if(detResult.isSuccess()){ for (DetectionInfo detectionInfo : detResult.getData().getDetectionInfoList()) { @@ -267,7 +288,7 @@ public class LivenessDetDemo { try { LivenessDetModel livenessDetModel = getLivenessDetModel(); //视频路径 - R livenessStatus = livenessDetModel.detectVideo("video.mp4"); + R livenessStatus = livenessDetModel.detectVideo("/Users/wenjie/Documents/idea_workplace/SmartJavaAI-Demo/src/main/resources/girl.mp4"); if (livenessStatus.isSuccess()){ log.info("识别结果：{}", JSONObject.toJSONString(livenessStatus.getData())); }else{ @@ -282,7 +303,7 @@ public class LivenessDetDemo { * 摄像头活体检测 * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 */ - @Test +// @Test public void testLivenessDetectCamera(){ try { LivenessDetModel livenessDetModel = getLivenessDetModel(); @@ -321,7 +342,7 @@ public class LivenessDetDemo { JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); } ViewerFrame frame = new ViewerFrame(width, height); - ImageFactory factory = ImageFactory.getInstance(); + SmartImageFactory factory = SmartImageFactory.getInstance(); Size size = new Size(width, height); while (capture.isOpened()) { @@ -330,9 +351,8 @@ public class LivenessDetDemo { } Mat resizeImage = new Mat(); Imgproc.resize(image, resizeImage, size); - Image img = factory.fromImage(resizeImage); - BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); - R detectedResult = livenessDetModel.detect(bufferedImage); + Image img = factory.fromMat(resizeImage); + R detectedResult = livenessDetModel.detect(img); if(!detectedResult.isSuccess()){ log.debug("识别失败：{}", detectedResult.getMessage()); continue; @@ -341,11 +361,10 @@ public class LivenessDetDemo { DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); Color color = detectionInfo.getFaceInfo().getLivenessStatus().getStatus() == LivenessStatus.LIVE ? Color.GREEN : Color.RED; String text = detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription() + ":" + detectionInfo.getFaceInfo().getLivenessStatus().getScore(); - ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, color); + ImageUtils.drawRectAndText(img, detectionRectangle, text); } - frame.showImage(bufferedImage); + frame.showImage(ImageUtils.toBufferedImage(img)); } - capture.release(); System.exit(0); } catch (Exception e) { diff --git a/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java index 714b59e..a297908 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java @@ -1,5 +1,8 @@ package smartai.examples.face.quality; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; @@ -23,16 +26,19 @@ import cn.smartjavaai.face.model.quality.FaceQualityModel; import cn.smartjavaai.face.utils.FaceUtils; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; /** * 人脸质量评估 demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -41,6 +47,14 @@ public class FaceQualityDetDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + //将图片处理的底层引擎切换为 OpenCV + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + @@ -52,7 +66,7 @@ public class FaceQualityDetDemo { QualityConfig config = new QualityConfig(); config.setModelEnum(QualityModelEnum.SEETA_FACE6_MODEL); //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + config.setModelPath("C:/Users/DengWenJie/Downloads/sf3.0_models/sf3.0_models"); config.setDevice(device); return FaceQualityModelFactory.getInstance().getModel(config); } @@ -64,7 +78,7 @@ public class FaceQualityDetDemo { */ public FaceDetModel getFaceDetModel() { //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; + String modelPath = "C:/Users/DengWenJie/Downloads/sf3.0_models/sf3.0_models"; FaceDetConfig faceDetectModelConfig = new FaceDetConfig(); faceDetectModelConfig.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); faceDetectModelConfig.setModelPath(modelPath); @@ -81,8 +95,9 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); @@ -114,8 +129,9 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); @@ -147,8 +163,8 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); - //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); @@ -180,8 +196,9 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); @@ -214,8 +231,9 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); @@ -248,8 +266,9 @@ public class FaceQualityDetDemo { try { FaceQualityModel faceQualityModel = getFaceQualityModel(); FaceDetModel faceDetModel = getFaceDetModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/iu_1.jpg"); //人脸检测 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); R detectionResponse = faceDetModel.detect(image); if(detectionResponse.isSuccess()){ log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); diff --git a/examples/face-example/src/test/python/model.py b/examples/face-example/src/test/python/model.py new file mode 100644 index 0000000..a3fd8a1 --- /dev/null +++ b/examples/face-example/src/test/python/model.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file +# except in compliance with the License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" +# BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for +# the specific language governing permissions and limitations under the License. +""" +PyTorch resnet18 pre/post processing example. +""" + +import json +import logging +import os +from typing import Optional, Any +import sklearn + +import torch +import torch.nn.functional as F +from torchvision import transforms + +from djl_python import Input +from djl_python import Output + + +class Processing(object): + + def __init__(self): + self.topK = 5 + self.image_processing = None + self.mapping = None + self.initialized = False + + def initialize(self, properties: dict): + """ + Initialize model. + """ + self.image_processing = transforms.Compose([ + transforms.Resize(112), + transforms.CenterCrop(112), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + #self.mapping = self.load_label_mapping("index_to_name.json") + self.initialized = True + + def preprocess(self, inputs: Input) -> Output: + outputs = Output() + try: + batch = inputs.get_batches() + images = [] + for i, item in enumerate(batch): + image = self.image_processing(item.get_as_image()) + images.append(image) + images = torch.stack(images) + outputs.add_as_numpy(images.detach().numpy()) + outputs.add_property("content-type", "tensor/ndlist") + except Exception as e: + logging.exception("pre-process failed") + # error handling + outputs = Output().error(str(e)) + + return outputs + + def postprocess(self, inputs: Input) -> Output: + outputs = Output() + try: + data = inputs.get_as_numpy(0)[0] + item = torch.from_numpy(data) + print("data shape:", item.shape) + embedding = sklearn.preprocessing.normalize(item).flatten() + outputs.add(embedding) + except Exception as e: + logging.exception("post-process failed") + # error handling + outputs = Output().error(str(e)) + + return outputs + + @staticmethod + def load_label_mapping(mapping_file_path: Any) -> dict: + if not os.path.isfile(mapping_file_path): + raise Exception('mapping file not found: ' + mapping_file_path) + + with open(mapping_file_path) as f: + mapping = json.load(f) + if not isinstance(mapping, dict): + raise Exception('mapping file should be in "class":"label" format') + + for key, value in mapping.items(): + new_value = value + if isinstance(new_value, list): + new_value = value[-1] + if not isinstance(new_value, str): + raise Exception( + 'labels in mapping must be either str or [str]') + mapping[key] = new_value + return mapping + + +_service = Processing() + + +def preprocess(inputs: Input) -> Output: + return _service.preprocess(inputs) + + +def postprocess(inputs: Input) -> Output: + return _service.postprocess(inputs) + + +def handle(inputs: Input) -> Optional[Output]: + """ + Default handler function + """ + if not _service.initialized: + # stateful model + _service.initialize(inputs.get_properties()) + + return None diff --git a/examples/objectdetection-example/README.md b/examples/objectdetection-example/README.md deleted file mode 100644 index 45b23eb..0000000 --- a/examples/objectdetection-example/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# 目标检测示例 - - -## 📁 项目结构 - -``` - -objectdetection-example/ -├── src/ -│ ├── main/ -│ │ ├── java/ -│ │ │ └── smartai/examples/objectdetection/ -│ │ │ ├── ObjectDetection.java -│ │ │ └── ViewerFrame.java - -``` - - ---- - -## 🧩 功能模块说明 - -### 1. 目标检测 [ObjectDetection.java] -- **功能**：核心目标检测类，包含多个测试方法，展示了如何使用不同的模型进行目标检测 - ---- - - -## ⚙️ 配置要求 - -- **运行环境**： - - JDK 1.8 或更高版本 - - IntelliJ IDEA 推荐作为开发 IDE -- **依赖库**： - - OpenCV、DJL、SmartJavaAI SDK -- **模型路径**： - - 所有模型需下载并配置正确的路径（参考各 demo 注释中的链接） - ---- - -## 🚀 快速开始 - -1. 克隆项目到本地： - -2. 导入项目至 IntelliJ IDEA。 - -3. 根据需要修改模型路径（见各 demo 中注释）。 - -4. 运行对应的 JUnit 测试类方法即可体验各项功能。 - ---- - -## 📄 文档 - -有关完整使用说明，请查阅 SmartJavaAI 官方文档： -[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) - ---- diff --git a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java deleted file mode 100644 index 14be538..0000000 --- a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java +++ /dev/null @@ -1,260 +0,0 @@ -package smartai.examples.objectdetection; - -import ai.djl.Application; -import ai.djl.MalformedModelException; -import ai.djl.modality.cv.Image; -import ai.djl.modality.cv.ImageFactory; -import ai.djl.modality.cv.output.*; -import ai.djl.modality.cv.output.Rectangle; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.repository.zoo.ModelZoo; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; -import cn.smartjavaai.common.entity.DetectionInfo; -import cn.smartjavaai.common.entity.DetectionRectangle; -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.R; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.common.utils.OpenCVUtils; -import cn.smartjavaai.objectdetection.config.DetectorModelConfig; -import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; -import cn.smartjavaai.objectdetection.exception.DetectionException; -import cn.smartjavaai.objectdetection.model.DetectorModel; -import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import nu.pattern.OpenCV; -import org.junit.Assert; -import org.junit.Test; -import org.opencv.core.Mat; -import org.opencv.core.Size; -import org.opencv.imgproc.Imgproc; -import org.opencv.videoio.VideoCapture; -import org.opencv.videoio.Videoio; - -import javax.imageio.ImageIO; -import javax.swing.*; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.*; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/** - * 目标检测模型demo - * 支持功能：目标检测 - * 模型下载地址：https://pan.baidu.com/s/10aTOLBlR6EG-sq6g0OkAWg?pwd=1234 提取码: 1234 - * @author dwj - */ -@Slf4j -public class ObjectDetection { - - - //设备类型 - public static DeviceEnum device = DeviceEnum.CPU; - - - - /** - * 使用默认模型检测：YOLO11N - */ - @Test - public void objectDetection(){ - //默认cpu - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/object_detection.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 指定模型检测(19种模型可选) - */ - @Test - public void objectDetection2(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.SSD_300_RESNET50);//检测模型，目前支持19种预置模型 - // 指定允许的类别 -// config.setAllowedClasses(Arrays.asList("person")); - //指定返回检测数量 - config.setTopK(100); - config.setDevice(device); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测并绘制检测结果 - */ - @Test - public void objectDetectionAndDraw(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - detectorModel.detectAndDraw("src/main/resources/object_detection.jpg","output/object_detection_detected.png"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测并绘制检测结果,返回BufferedImage - */ - @Test - public void objectDetectionAndDraw2(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - String imagePath = "src/main/resources/object_detection.jpg"; - BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - BufferedImage detectedImage = detectorModel.detectAndDraw(image); - Assert.assertNotNull("detectedImage null", detectedImage); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - - /** - * 使用yolo官方模型检测物品识别 - */ - @Test - public void objectDetectionWithOfficialModel(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - config.setThreshold(0.3f); - //也支持YoloV8：YOLOV8_OFFICIAL 模型可以从文档中提供的地址下载 - config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL);//检测模型，目前支持19种模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("/Users/xxx/Documents/yolov12n.onnx"); - config.setDevice(device); - //一定要将yolo官方的类别文件：synset.txt（文档中下载）放在模型同目录下，否则报错 - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 使用自己训练的模型检测 - */ - @Test - public void objectDetectionWithCustomModel(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - //也支持YoloV8：YOLOV8_CUSTOM 模型需要自己训练，训练教程可以查看文档 - config.setModelEnum(DetectorModelEnum.YOLOV12_CUSTOM);//自定义YOLOV12模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("/Users/xxx/Documents/develop/fire_model/best.onnx"); - config.putCustomParam("width", 640);//resize 宽 - config.putCustomParam("height", 640);// resize 高 - config.putCustomParam("nmsThreshold", 0.5f); - // 指定允许的类别 -// config.setAllowedClasses(Arrays.asList("person")); - //指定返回检测数量 - config.setTopK(100); - config.setDevice(device); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - /** - * 摄像头目标检测 - * 注意事项：如果视频比较卡，可以使用轻量的检测模型 - */ - @Test - public void testDetectCamera(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - OpenCV.loadShared(); - VideoCapture capture = new VideoCapture(0); - if (!capture.isOpened()) { - System.out.println("No camera detected"); - return; - } - - double ratio = - capture.get(Videoio.CAP_PROP_FRAME_WIDTH) - / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); - Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); - int height = (int) (screenSize.height * 0.65f); - int width = (int) (height * ratio); - if (width > screenSize.width) { - width = screenSize.width; - } - - Mat image = new Mat(); - boolean captured = false; - for (int i = 0; i < 10; ++i) { - captured = capture.read(image); - if (captured) { - break; - } - - try { - Thread.sleep(50); - } catch (InterruptedException ignore) { - // ignore - } - } - if (!captured) { - JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); - } - ViewerFrame frame = new ViewerFrame(width, height); - ImageFactory factory = ImageFactory.getInstance(); - Size size = new Size(width, height); - - while (capture.isOpened()) { - if (!capture.read(image)) { - break; - } - Mat resizeImage = new Mat(); - Imgproc.resize(image, resizeImage, size); - Image img = factory.fromImage(resizeImage); - BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); - DetectionResponse detectedResult = detectorModel.detect(bufferedImage); - if (Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().size() == 0){ - log.debug("未检测到物体"); - continue; - } - for(DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){ - DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); - String text = detectionInfo.getObjectDetInfo().getClassName(); - ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.RED); - } - frame.showImage(bufferedImage); - } - - capture.release(); - System.exit(0); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - -} diff --git a/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF b/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF deleted file mode 100644 index c9a4764..0000000 --- a/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF +++ /dev/null @@ -1,3 +0,0 @@ -Manifest-Version: 1.0 -Main-Class: smartai.examples.objectdetection.ObjectDetection - diff --git a/examples/ocr-examples/README.md b/examples/ocr-examples/README.md index 4b395a1..b30c95c 100644 --- a/examples/ocr-examples/README.md +++ b/examples/ocr-examples/README.md @@ -9,11 +9,16 @@ src ├── main │ ├── java │ │ └── smartai/examples/ocr -│ │ ├── OcrDetectionDemo.java # 文本检测示例 -│ │ ├── OcrDirectionDetDemo.java # 文本方向检测示例 -│ │ └── OcrRecognizeDemo.java # 文本识别示例 +│ │ ├── common +│ │ │ ├── OcrDetectionDemo.java # 文本检测示例 +│ │ │ ├── OcrDirectionDetDemo.java # 文本方向检测示例 +│ │ │ └── OcrRecognizeDemo.java # 文本识别示例 +│ │ └── table +│ │ └── TableRecDemo.java # 表格识别示例 +│ │ └── plate +│ │ └── PlateRecDemo.java # 车牌识别示例 │ └── resources -│ ├── logback.xml # 日志配置文件 +│ ├── logback.xml # 日志配置文件 └── test @@ -22,24 +27,6 @@ src --- -## 🧩 功能说明 - -### 1. 文本检测 - [OcrDetectionDemo] - -- **功能**：检测图像中的文本区域，仅返回文本框位置，不识别文字内容。 - - -### 2. 文本方向检测 - [OcrDirectionDetDemo] - -- **功能**：在文本检测基础上，判断文本整体方向（0°, 90°, 180°, 270°）。 - -### 3. 文本识别 - [OcrRecognizeDemo] - -- **功能**：对检测到的文本区域进行文字识别，支持简体中文、繁体中文、英文、日文等。 -- **流程**： - - 文本检测 → 文本识别（或加上方向矫正） - ---- ## ⚙️ 配置要求 @@ -56,13 +43,18 @@ src ## 🚀 快速开始 -1. 克隆项目到本地： +如果你只想运行某个示例，请按以下方式操作： -2. 导入项目至 IntelliJ IDEA。 +1. 打开 IDEA（或你喜欢的 IDE） +2. 选择 **“Open”**，然后仅导入 `examples` 目录下对应的示例项目，例如： -3. 根据需要修改模型路径（见各 demo 中注释）。 + ``` + examples/ocr-example + ``` +3. IDEA 会自动识别并加载依赖。若首次导入，请等待 Maven 下载依赖完成。 +4. 请从我们提供的 百度网盘 中下载模型及其附带文件，并在示例代码中将模型路径修改为您本地的实际路径。 +5. 可通过查看每个 Java 文件顶部的注释了解对应功能，或参考 README 文件中对各 Java 文件功能的说明，运行相应的测试方法进行体验。 -4. 运行对应的 JUnit 测试类方法即可体验各项功能。 --- diff --git a/examples/ocr-examples/pom.xml b/examples/ocr-examples/pom.xml index f89411e..ce4ec26 100644 --- a/examples/ocr-examples/pom.xml +++ b/examples/ocr-examples/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.22 + 1.0.27 smartai.examples.ocr.common.OcrRecognizeDemo @@ -34,7 +34,7 @@ cn.smartjavaai - smartjavaai-bom + bom ${smartjavaai.version} pom @@ -50,16 +50,13 @@ commons-cli 1.9.0 + commons-io commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +70,6 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - com.alibaba @@ -96,7 +88,7 @@ cn.smartjavaai - smartjavaai-ocr + ocr @@ -181,6 +173,13 @@ 2.5.1 runtime + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + @@ -220,45 +219,6 @@ - - - org.bytedeco - javacpp - ${javacv.version} - ${javacv.platform.linux-arm64} - - - - org.bytedeco - ffmpeg - 6.1.1-1.5.10 - ${javacv.platform.linux-arm64} - - - - org.bytedeco - openblas - 0.3.26-1.5.10 - ${javacv.platform.linux-arm64} - - - - org.bytedeco - opencv - 4.9.0-1.5.10 - ${javacv.platform.linux-arm64} - - - - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.5.1 - runtime - - - - @@ -287,16 +247,21 @@ + + + + + + + + + + + + - aliyunmaven - 阿里云公共仓库 - https://maven.aliyun.com/repository/public - - true - - - false - + central + https://repo1.maven.org/maven2/ diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java index e64cc30..1b70918 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java @@ -1,6 +1,8 @@ package smartai.examples.ocr.common; import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.utils.ImageUtils; @@ -14,37 +16,49 @@ import cn.smartjavaai.ocr.model.common.detect.OcrCommonDetModel; import cn.smartjavaai.ocr.utils.OcrUtils; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; import org.opencv.core.Mat; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.util.List; /** * OCR 文本检测 示例 * 模型下载地址：https://pan.baidu.com/s/15Noz2xHQzqMQSl1B19BobQ?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j public class OcrDetectionDemo { + + //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** * 获取文本检测模型 * @return */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） - config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); + config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); config.setDevice(device); return OcrModelFactory.getInstance().getDetModel(config); } @@ -61,7 +75,9 @@ public class OcrDetectionDemo { public void detect(){ try { OcrCommonDetModel model = getDetectionModel(); - List boxes = model.detect("src/main/resources/ocr_1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_1.jpg"); + List boxes = model.detect(image); log.info("OCR检测结果：{}", JSONObject.toJSONString(boxes)); } catch (Exception e) { e.printStackTrace(); @@ -85,6 +101,26 @@ public class OcrDetectionDemo { } } + /** + * 文本检测并绘制结果 + * 检测图像中的文本区域，仅检测文本框位置，不识别文字内容 + * 注意事项： + * 1、批量检测时，模型应统一放在外层 try 中使用，避免重复加载，自动释放资源更安全。 + * 2、模型文件需要放在单独文件夹 + */ + @Test + public void detectAndDraw2(){ + try { + OcrCommonDetModel model = getDetectionModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_1.jpg"); + Image resultImage = model.detectAndDraw(image); + ImageUtils.save(resultImage, "output/ocr_1_detected2.jpg"); + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 批量文本检测：批量检测要求图片宽高一致 @@ -98,7 +134,7 @@ public class OcrDetectionDemo { try { OcrCommonDetModel model = getDetectionModel(); //批量检测要求图片宽高一致 - String folderPath = "/Users/xxx/Downloads/testing33"; + String folderPath = "/Users/wenjie/Downloads/testing33"; //读取文件夹中所有图片 List images = ImageUtils.readImagesFromFolder(folderPath); List> ocrResult = model.batchDetectDJLImage(images); diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java index 5e93842..e665fae 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java @@ -1,6 +1,10 @@ package smartai.examples.ocr.common; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.ocr.config.DirectionModelConfig; import cn.smartjavaai.ocr.config.OcrDetModelConfig; import cn.smartjavaai.ocr.entity.OcrBox; @@ -12,15 +16,17 @@ import cn.smartjavaai.ocr.model.common.detect.OcrCommonDetModel; import cn.smartjavaai.ocr.model.common.direction.OcrDirectionModel; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; import java.util.List; /** * OCR 行文本方向检测 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class OcrDirectionDetDemo { @@ -28,16 +34,22 @@ public class OcrDirectionDetDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** * 获取方向检测模型 * @return */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setModelPath("/Users/xxx/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); + directionModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); directionModelConfig.setDevice(device); directionModelConfig.setTextDetModel(getDetectionModel()); return OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); @@ -49,10 +61,10 @@ public class OcrDirectionDetDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） - config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); + config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); config.setDevice(device); return OcrModelFactory.getInstance().getDetModel(config); } @@ -69,7 +81,9 @@ public class OcrDirectionDetDemo { public void detect(){ try { OcrDirectionModel directionModel = getDirectionModel(); - List itemList = directionModel.detect("src/main/resources/ocr_1.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_1.jpg"); + List itemList = directionModel.detect(image); log.info("OCR方向检测结果1：{}", JSONObject.toJSONString(itemList)); } catch (Exception e) { e.printStackTrace(); @@ -93,6 +107,25 @@ public class OcrDirectionDetDemo { } } + /** + * 文本检测并绘制结果 + * 流程：文本检测 -> 方向分类 + * 检测图像中的文本区域，仅检测文本框位置，不识别文字内容 + * 模型需要放在单独文件夹 + */ + @Test + public void detectAndDraw2(){ + try { + OcrDirectionModel directionModel = getDirectionModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_1.jpg"); + Image resultImage = directionModel.detectAndDraw(image); + ImageUtils.save(resultImage, "output/ocr_1_detected4.jpg"); + } catch (Exception e) { + e.printStackTrace(); + } + } + } diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java index 263d5bd..45d5b47 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java @@ -1,7 +1,13 @@ package smartai.examples.ocr.common; import ai.djl.modality.cv.Image; +import ai.djl.util.JsonUtils; +import cn.hutool.core.img.ImgUtil; +import cn.hutool.core.io.FileUtil; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.BufferedImageUtils; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.ocr.config.DirectionModelConfig; import cn.smartjavaai.ocr.config.OcrDetModelConfig; @@ -18,16 +24,22 @@ import cn.smartjavaai.ocr.model.common.direction.OcrDirectionModel; import cn.smartjavaai.ocr.model.common.recognize.OcrCommonRecModel; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; /** * OCR 文本识别 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class OcrRecognizeDemo { @@ -35,31 +47,74 @@ public class OcrRecognizeDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 + //Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** - * 获取通用识别模型（不带方向矫正） + * 获取通用识别模型（高精确度模型） + * 注意事项：高精度模型，识别准确度高，速度慢 * @return */ - public OcrCommonRecModel getRecModel(){ + public OcrCommonRecModel getProRecModel(){ OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 - recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath + recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_SERVER_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); + recModelConfig.setRecModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); recModelConfig.setDevice(device); - recModelConfig.setTextDetModel(getDetectionModel()); + recModelConfig.setTextDetModel(getProDetectionModel()); + recModelConfig.setDirectionModel(getDirectionModel()); return OcrModelFactory.getInstance().getRecModel(recModelConfig); } /** - * 获取文本检测模型 + * 获取通用识别模型（极速模型） + * 注意事项：极速模型，识别准确度低，速度快 * @return */ - public OcrCommonDetModel getDetectionModel() { + public OcrCommonRecModel getFastRecModel(){ + OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath + recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); + //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） + recModelConfig.setRecModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); + recModelConfig.setDevice(device); + recModelConfig.setTextDetModel(getFastDetectionModel()); + return OcrModelFactory.getInstance().getRecModel(recModelConfig); + } + + + /** + * 获取文本检测模型(极速模型) + * 注意事项：极速模型，识别准确度低，速度快 + * @return + */ + public OcrCommonDetModel getFastDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） - config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); + config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); + config.setDevice(device); + return OcrModelFactory.getInstance().getDetModel(config); + } + + /** + * 获取文本检测模型(高精确度模型) + * 注意事项：高精度模型，识别准确度高，速度慢 + * @return + */ + public OcrCommonDetModel getProDetectionModel() { + OcrDetModelConfig config = new OcrDetModelConfig(); + //指定检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_SERVER_DET_MODEL); + //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） + config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); config.setDevice(device); return OcrModelFactory.getInstance().getDetModel(config); } @@ -70,31 +125,15 @@ public class OcrRecognizeDemo { */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setModelPath("/Users/xxx/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); + directionModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); directionModelConfig.setDevice(device); return OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); } - /** - * 获取通用识别模型(带方向矫正) - * @return - */ - public OcrCommonRecModel getRecModelWithDirection() { - OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 - recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); - //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); - recModelConfig.setDevice(device); - recModelConfig.setTextDetModel(getDetectionModel()); - recModelConfig.setDirectionModel(getDirectionModel()); - return OcrModelFactory.getInstance().getRecModel(recModelConfig); - } - /** * 文本识别 @@ -107,10 +146,12 @@ public class OcrRecognizeDemo { @Test public void recognize(){ try { - OcrCommonRecModel recModel = getRecModel(); + OcrCommonRecModel recModel = getFastRecModel(); //不带方向矫正，分行返回文本 OcrRecOptions options = new OcrRecOptions(false, true); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_2.jpg",options); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_1.jpg"); + OcrInfo ocrInfo = recModel.recognize(image, options); log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); } catch (Exception e) { e.printStackTrace(); @@ -129,8 +170,10 @@ public class OcrRecognizeDemo { @Test public void recognizeHandWriting(){ try { - OcrCommonRecModel recModel = getRecModel(); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/handwriting_1.jpg",new OcrRecOptions()); + OcrCommonRecModel recModel = getFastRecModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/handwriting_1.jpg"); + OcrInfo ocrInfo = recModel.recognize(image, new OcrRecOptions()); log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); } catch (Exception e) { e.printStackTrace(); @@ -149,10 +192,12 @@ public class OcrRecognizeDemo { @Test public void recognize2(){ try { - OcrCommonRecModel recModel = getRecModelWithDirection(); + OcrCommonRecModel recModel = getFastRecModel(); //带方向矫正，分行返回文本 OcrRecOptions options = new OcrRecOptions(true, true); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_3.jpg",options); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/ocr_3.jpg"); + OcrInfo ocrInfo = recModel.recognize(image, options); log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); } catch (Exception e) { e.printStackTrace(); @@ -172,7 +217,7 @@ public class OcrRecognizeDemo { @Test public void recognizeAndDraw(){ try { - OcrCommonRecModel recModel = getRecModelWithDirection(); + OcrCommonRecModel recModel = getFastRecModel(); int fontSize = 18; recModel.recognizeAndDraw("src/main/resources/general_ocr_002.png", "output/ocr_4_recognized.jpg", fontSize, new OcrRecOptions()); } catch (Exception e) { @@ -180,6 +225,28 @@ public class OcrRecognizeDemo { } } + @Test + public void recognizeAndDraw2(){ + try { + OcrCommonRecModel recModel = getFastRecModel(); + int fontSize = 18; + //创建保存路径 + Path inputImagePath = Paths.get("src/main/resources/general_ocr_002.png"); + Path imageOutputPath = Paths.get("output/ocr_5_recognized.jpg"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(inputImagePath); + OcrInfo ocrInfo = recModel.recognizeAndDraw(image, fontSize, new OcrRecOptions()); + log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + //保存绘制结果 + if(ocrInfo != null && ocrInfo.getDrawnImage() != null){ + ImageUtils.save(ocrInfo.getDrawnImage(), imageOutputPath.toAbsolutePath().toString()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** * 批量识别 * 注意事项： @@ -189,7 +256,7 @@ public class OcrRecognizeDemo { @Test public void batchRecognize(){ try { - OcrCommonRecModel recModel = getRecModelWithDirection(); + OcrCommonRecModel recModel = getFastRecModel(); //批量检测要求图片宽高一致 String folderPath = "/Users/xxx/Downloads/testing33"; //读取文件夹中所有图片 diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java index 7cb59ae..0dd1947 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java @@ -1,6 +1,9 @@ package smartai.examples.ocr.plate; +import ai.djl.modality.cv.Image; import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.utils.ImageUtils; @@ -13,12 +16,20 @@ import cn.smartjavaai.ocr.factory.PlateModelFactory; import cn.smartjavaai.ocr.model.plate.PlateDetModel; import cn.smartjavaai.ocr.model.plate.PlateRecModel; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; import java.util.List; /** + * 车牌识别demo + * 模型下载地址：https://pan.baidu.com/s/1YEP56UqYcL-Op80M6JAreA?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -27,14 +38,23 @@ public class PlateRecDemo { //设备类型 public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass + public static void beforeAll() throws IOException { + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + /** * 获取车牌检测模型 * @return */ public PlateDetModel getPlateDetModel() { PlateDetModelConfig config = new PlateDetModelConfig(); + //车牌检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(PlateDetModelEnum.YOLOV5); - config.setModelPath("/Users/xxx/Documents/develop/model/plate/yolov5_plate_detect.onnx"); + //下载模型并替换本地路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/plate/yolov5_plate_detect.onnx"); config.setDevice(device); return PlateModelFactory.getInstance().getDetModel(config); } @@ -45,16 +65,26 @@ public class PlateRecDemo { */ public PlateRecModel getPlateRecModel() { PlateRecModelConfig recModelConfig = new PlateRecModelConfig(); + //车牌识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setModelEnum(PlateRecModelEnum.PLATE_REC_CRNN); - recModelConfig.setModelPath("/Users/xxx/Documents/develop/model/plate/plate_rec_color.onnx"); + //下载模型并替换本地路径 + recModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/plate/plate_rec_color.onnx"); + //指定车牌检测模型 recModelConfig.setPlateDetModel(getPlateDetModel()); + recModelConfig.setDevice(device); return PlateModelFactory.getInstance().getRecModel(recModelConfig); } + /** + * 车牌识别 + */ @Test - public void testDetect() { + public void testDetect() throws IOException { PlateRecModel plateRecModel = getPlateRecModel(); - R> result = plateRecModel.recognize("src/main/resources/plate/Quicker_20220930_180856.png"); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile("src/main/resources/plate/Quicker_20220930_180856.png"); + //识别车号 + R> result = plateRecModel.recognize(image); if(result.isSuccess()){ log.info("车牌识别结果：{}", JsonUtils.toJson(result.getData())); }else{ @@ -62,9 +92,13 @@ public class PlateRecDemo { } } + /** + * 车牌识别及绘制结果 + */ @Test public void recognizeAndDraw() { PlateRecModel plateRecModel = getPlateRecModel(); + //识别车号并绘制结果 R result = plateRecModel.recognizeAndDraw("src/main/resources/plate/single_green.jpg", "output/plate_recognized2.jpg"); if(result.isSuccess()){ log.info("车牌识别成功"); @@ -74,5 +108,28 @@ public class PlateRecDemo { } + /** + * 车牌识别及绘制结果 + */ + @Test + public void recognizeAndDraw2() { + try { + PlateRecModel plateRecModel = getPlateRecModel(); + String imagePath = "src/main/resources/plate/Quicker_20220930_180856.png"; + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(imagePath); + //可以根据后续业务场景使用detectedImage + R detectedImage = plateRecModel.recognizeAndDraw(image); + if(detectedImage.isSuccess()){ + log.info("车牌识别成功"); + ImageUtils.save(detectedImage.getData(), "output/plate_recognized3.jpg"); + }else{ + log.error("车牌识别失败：{}", detectedImage.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java index 0aba097..942d1bc 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java @@ -2,6 +2,8 @@ package smartai.examples.ocr.table; import ai.djl.modality.cv.Image; import cn.hutool.core.io.FileUtil; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.utils.ImageUtils; @@ -21,24 +23,36 @@ import cn.smartjavaai.ocr.model.table.TableRecognizer; import cn.smartjavaai.ocr.model.table.TableStructureModel; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; import org.junit.Test; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; /** - * OCR 文本识别 示例 + * OCR 表格识别 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class TableRecDemo { + + @BeforeClass + public static void beforeAll() throws IOException { + SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV); + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + //设备类型 public static DeviceEnum device = DeviceEnum.CPU; @@ -48,10 +62,10 @@ public class TableRecDemo { */ public OcrCommonRecModel getRecModel(){ OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); + recModelConfig.setRecModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); recModelConfig.setDevice(device); recModelConfig.setTextDetModel(getDetectionModel()); return OcrModelFactory.getInstance().getRecModel(recModelConfig); @@ -63,11 +77,10 @@ public class TableRecDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） - config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); -// config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); + config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); config.setDevice(device); return OcrModelFactory.getInstance().getDetModel(config); } @@ -78,10 +91,10 @@ public class TableRecDemo { */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setModelPath("/Users/xxx/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); + directionModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); directionModelConfig.setDevice(device); return OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); } @@ -92,10 +105,10 @@ public class TableRecDemo { */ public TableStructureModel getTableStructureModel(){ TableStructureConfig config = new TableStructureConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(TableStructureModelEnum.SLANET_PLUS); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - config.setModelPath("/Users/xxx/Documents/develop/model/ocr/slanet-plus/slanet-plus.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/ocr/slanet-plus/slanet-plus.onnx"); // config.setModelPath("/Users/xxx/Documents/develop/model/ocr/SLANet_infer/SLANet.onnx"); config.setDevice(device); return TableRecModelFactory.getInstance().getTableStructureModel(config); @@ -103,6 +116,7 @@ public class TableRecDemo { + /** * 表格识别 * 仅支持简单表格 @@ -112,7 +126,7 @@ public class TableRecDemo { * 2、模型文件需要放在单独文件夹 */ @Test - public void recognize(){ + public void recognize2(){ try { TableStructureModel tableStructureModel = getTableStructureModel(); OcrCommonDetModel detModel = getDetectionModel(); @@ -125,7 +139,8 @@ public class TableRecDemo { // .withDirectionModel(getDirectionModel()) //如果表格中存在旋转的文字，可以使用方向分类模型 .withTextRecModel(recModel).build(); String imagePath = "src/main/resources/table/table_ch1.png"; - BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(imagePath); R result = tableRecognizer.recognize(image); if(result.isSuccess()){ log.info("result: {}", result.getData().getHtml()); @@ -133,9 +148,12 @@ public class TableRecDemo { Path outputPath = Paths.get("output/table_ch2_result.html"); FileUtil.writeUtf8String(result.getData().getHtml(), outputPath.toAbsolutePath().toString()); //绘制表格结构 - tableRecognizer.drawTable(result.getData(), image, "output/table_ch2_result.jpg"); + Image resultImage = tableRecognizer.drawTable(result.getData(), image); + ImageUtils.save(resultImage, "output/table_ch2_result.jpg"); //导出excel，如果导出失败，可能是因为表格结果识别的结果是错乱的 - tableRecognizer.exportExcel(result.getData().getHtml(), "output/table_ch2_result.xls"); + try (OutputStream out = Files.newOutputStream(Paths.get("output/table_ch2_result2.xls"))) { + tableRecognizer.exportExcel(result.getData().getHtml(), out); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/examples/objectdetection-example/.gitignore b/examples/speech-examples/.gitignore similarity index 100% rename from examples/objectdetection-example/.gitignore rename to examples/speech-examples/.gitignore diff --git a/examples/speech-examples/README.md b/examples/speech-examples/README.md new file mode 100644 index 0000000..d5b2d84 --- /dev/null +++ b/examples/speech-examples/README.md @@ -0,0 +1,38 @@ +# 语音识别ASR+语音合成TTS示例 + +## 📁 项目结构 + +``` +src +├── main +│ ├── java +│ │ └── smartai/examples/speech +│ │ ├── asr +│ │ │ ├── SherpaAsrDemo.java # sherpa-onnx 语音识别ASR 示例 +│ │ │ ├── SpeechRecognizeDemo.java # 其他模型的 语音识别ASR 示例 +│ │ └── tts +│ │ └── TtsDemo.java # tts语音合成示例 +``` + +## 🚀 快速开始 + +如果你只想运行某个示例，请按以下方式操作： + +1. 打开 IDEA（或你喜欢的 IDE） +2. 选择 **“Open”**，然后仅导入 `examples` 目录下对应的示例项目，例如： + + ``` + examples/speech-example + ``` +3. IDEA 会自动识别并加载依赖。若首次导入，请等待 Maven 下载依赖完成。 +4. 请从我们提供的 百度网盘 中下载模型及其附带文件，并在示例代码中将模型路径修改为您本地的实际路径。 +5. 可通过查看每个 Java 文件顶部的注释了解对应功能，或参考 README 文件中对各 Java 文件功能的说明，运行相应的测试方法进行体验。 + +--- + +## 📄 文档 + +有关完整使用说明，请查阅 SmartJavaAI 官方文档： +[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) + +--- diff --git a/examples/speech-examples/pom.xml b/examples/speech-examples/pom.xml new file mode 100644 index 0000000..e077903 --- /dev/null +++ b/examples/speech-examples/pom.xml @@ -0,0 +1,211 @@ + + + 4.0.0 + + cn.smartjavaai + examples-speech + 1.0.0-SNAPSHOT + + + 11 + 11 + UTF-8 + 1.0.27 + + smartai.examples.speech.asr.common.OcrRecognizeDemo + + + + + + + cn.smartjavaai + bom + ${smartjavaai.version} + pom + + import + + + + + + + + commons-cli + commons-cli + 1.9.0 + + + commons-io + commons-io + 2.17.0 + + + org.testng + testng + 7.10.2 + test + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + + com.alibaba + fastjson + 1.2.83 + + + + junit + junit + 4.13.2 + + + + + + + cn.smartjavaai + speech + + + com.microsoft.onnxruntime + onnxruntime + + + org.openpnp + opencv + + + ai.djl.huggingface + tokenizers + + + ai.djl.ml.xgboost + xgboost + + + + + + + ws.schild + jave-all-deps + 3.5.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + shade + + false + + + + ${exec.mainClass} + + + + + + + + + + + + + + + + + + + + + + + + + central + https://repo1.maven.org/maven2/ + + + + + + + + + + diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java new file mode 100644 index 0000000..ff814d7 --- /dev/null +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java @@ -0,0 +1,389 @@ +package smartai.examples.speech.asr; + +import ai.djl.modality.audio.Audio; +import ai.djl.modality.audio.AudioFactory; +import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.entity.Language; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.AsrResult; +import cn.smartjavaai.speech.asr.entity.WhisperParams; +import cn.smartjavaai.speech.asr.enums.AsrModelEnum; +import cn.smartjavaai.speech.asr.factory.SpeechRecognizerFactory; +import cn.smartjavaai.speech.asr.model.SpeechRecognizer; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.nio.file.Paths; + +/** + * 语音识别ASR demo + * sherpa-onnx模型及依赖库下载链接: + * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases + * 2、（推荐）ASR模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + * 3、百度网盘下载：https://pan.baidu.com/s/19p3WhVEM7dgdkvXFaeeAxg?pwd=1234 提取码: 1234 + * @author dwj + * @date 2025/10/23 + */ +@Slf4j +public class SherpaAsrDemo { + + + /** + * 语音识别：Sherpa Paraformer（中文） + */ + @Test + public void testSherpaParaformerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_PARAFORMER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14"); + config.setModelName("model.int8.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Dolphin（中文） + */ + @Test + public void testSherpaDolphinAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_DOLPHIN); + config.setModelName("model.int8.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa zipformer（中文） + */ + @Test + public void testSherpaZipformerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_ZIPFORMERCTC); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + config.setModelName("model.int8.onnx"); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa FireRedAsr（中英） + */ + @Test + public void testSherpaFireRedAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa SenseVoice （多语言模型） + */ + @Test + public void testSherpaSenseVoiceAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_SENSEVOICE); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17"); + config.setModelName("model.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa WenetCtc（多语言模型：粤语） + */ + @Test + public void testSherpaWenetCtcAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_WENETCTC); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10"); + config.setModelName("model.int8.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/test_wavs/yue-0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa FireRedAsr（方言：四川、天津、河南） + */ + @Test + public void testSherpaFireRedAsrSichuan() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3-sichuan.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Telespeech（方言-天津、河南、四川） + */ + @Test + public void testSherpaTelespeechAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_TELESPEECH); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.setModelName("model.int8.onnx"); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/test_wavs/4-tianjin.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Nemo（英文） + */ + @Test + public void testSherpaNemoAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_NEMO); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512"); + config.setModelName("model.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Moonshine（英文） + */ + @Test + public void testSherpaMoonshineAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_MOONSHINE); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 2); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + /** + * 语音识别：Sherpa Whisper（英文） + */ + @Test + public void testSherpaWhisperAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_WHISPER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Transducer（英文） + */ + @Test + public void testSherpaTransducerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_TRANSDUCER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12/test_wavs/1221-135766-0001.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Canary（德语-英文） + */ + @Test + public void testSherpaCanaryAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_CANARY); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/de.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + + + + + + + + +} diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java new file mode 100644 index 0000000..23795d8 --- /dev/null +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java @@ -0,0 +1,353 @@ +package smartai.examples.speech.asr; + +import ai.djl.util.JsonUtils; +import cn.hutool.core.io.FileUtil; +import cn.smartjavaai.common.entity.Language; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.AsrResult; +import cn.smartjavaai.speech.asr.entity.VoskParams; +import cn.smartjavaai.speech.asr.entity.WhisperParams; +import cn.smartjavaai.speech.asr.enums.AsrModelEnum; +import cn.smartjavaai.speech.asr.factory.SpeechRecognizerFactory; +import cn.smartjavaai.speech.asr.model.SpeechRecognizer; +import cn.smartjavaai.speech.asr.model.VoskRecognizer; +import cn.smartjavaai.speech.asr.model.WhisperRecognizer; +import io.github.givimad.whisperjni.WhisperFullParams; +import io.github.givimad.whisperjni.WhisperGrammar; +import io.github.givimad.whisperjni.WhisperSamplingStrategy; +import io.github.givimad.whisperjni.WhisperState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.junit.Test; +import org.vosk.Recognizer; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.TargetDataLine; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.file.Paths; + +/** + * 语音识别demo（Vosk、Whisper） + * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class SpeechRecognizeDemo { + + + + /** + * 获取Whisper模型 + * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 更多模型下载地址：https://huggingface.co/ggerganov/whisper.cpp/tree/main + * @return + */ + public SpeechRecognizer getWhisperRecognizer() { + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.WHISPER); + //模型下载地址：https://huggingface.co/ggerganov/whisper.cpp/tree/main + config.setModelPath("/Users/xxx/Documents/develop/model/speech/ggml-medium.bin"); + return SpeechRecognizerFactory.getInstance().getModel(config); + } + + + /** + * Whisper 语音识别 + * 多语言模型支持100种语言 + * 注意事项： + * 1、不支持centos7 + * 2、模型越大越准确 + * 3、暂不支持GPU使用，如需GPU使用需要自行编译：https://github.com/ggml-org/whisper.cpp/tree/master?tab=readme-ov-file#nvidia-gpu-support + */ + @Test + public void testWhisper() { + try { + SpeechRecognizer recognizer = getWhisperRecognizer(); + WhisperParams params = new WhisperParams(); + //语言：中文 + params.setLanguage(Language.ZH); + R result = recognizer.recognize("src/main/resources/speech_zh.mp3", params); + if (result.isSuccess()){ + log.info("识别成功:{}", JsonUtils.toJson(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Whisper 语音识别(使用个性化配置) + * 多语言模型支持100种语言 + * 注意事项： + * 1、不支持centos7 + * 2、模型越大越准确 + * 3、暂不支持GPU使用，如需GPU使用需要自行编译：https://github.com/ggml-org/whisper.cpp/tree/master?tab=readme-ov-file#nvidia-gpu-support + */ + @Test + public void testWhisperWithCustomConfig() { + try { + SpeechRecognizer recognizer = getWhisperRecognizer(); + WhisperParams params = new WhisperParams(); + //语言：中文 + params.setLanguage(Language.ZH); + /** + * 解码搜索策略类型： + * GREEDY - 贪婪解码，逐步选择概率最高的结果； + * BEAN_SEARCH - Beam 搜索，保留多个候选路径以提高准确性。 + */ + WhisperFullParams fullParams = new WhisperFullParams(WhisperSamplingStrategy.BEAN_SEARCH); + //语言 + fullParams.language = Language.ZH.getCode(); + //线程数，设为 0 表示使用最大核心数。 + fullParams.nThreads = 0; + //解码器使用的历史文本作为提示的最大 token 数。 + fullParams.nMaxTextCtx = 16384; + //解码起始偏移（毫秒） + fullParams.offsetMs = 0; + //解码持续时长（毫秒），超过此长度的音频将被截断 + fullParams.durationMs = 0; + //是否翻译为英文 + fullParams.translate = false; + // 初始提示，用于提供上下文或样例，帮助模型更准确地理解语音内容 + fullParams.initialPrompt = "简体中文"; + //禁用上下文链接，不使用前一段解码结果作为上下文 + fullParams.noContext = true; + //是否强制仅输出一个段落（适用于短语音） + fullParams.singleSegment = false; + //是否打印特殊标记 + fullParams.printSpecial = false; + //是否直接从 whisper.cpp 中打印结果（不推荐，建议使用回调方式替代） + fullParams.printRealtime = false; + //抑制非语音 token输出 + fullParams.suppressNonSpeechTokens = false; + //更多参数请查看官网：https://github.com/GiviMAD/whisper-jni/blob/33854520b1f0b3697106a7932a2fd64e8191bca9/src/main/java/io/github/givimad/whisperjni/WhisperFullParams.java + params.setParams(fullParams); + //建议上传 WAV 格式音频。其他格式将自动转换为 WAV，可能影响处理速度 + R result = recognizer.recognize("src/main/resources/speech_zh.mp3", params); + if (result.isSuccess()){ + log.info("识别成功:{}", JsonUtils.toJson(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Whisper 语音识别(使用Grammar语法规则) + * 多语言模型支持100种语言 + * 注意事项： + * 1、不支持centos7 + * 2、模型越大越准确 + * 3、暂不支持GPU使用，如需GPU使用需要自行编译：https://github.com/ggml-org/whisper.cpp/tree/master?tab=readme-ov-file#nvidia-gpu-support + */ + @Test + public void testWhisperWithGrammar() { + try { + WhisperRecognizer whisperRecognizer = (WhisperRecognizer)getWhisperRecognizer(); + //语法规则 + String grammarText = "root ::= \" And so, my fellow American, ask not what your country can do for you, ask what you can do for your country.\""; + try (WhisperGrammar grammar = whisperRecognizer.parseGrammar(grammarText)){ + WhisperParams params = new WhisperParams(); + WhisperFullParams fullParams = new WhisperFullParams(WhisperSamplingStrategy.BEAN_SEARCH); + //语言：英文 + fullParams.language = Language.EN.getCode(); + fullParams.grammar = grammar; + params.setParams(fullParams); + //建议上传 WAV 格式音频。其他格式将自动转换为 WAV，可能影响处理速度 + R result = whisperRecognizer.recognize("src/main/resources/jfk_en.wav", params); + if (result.isSuccess()){ + log.info("识别成功:{}", JsonUtils.toJson(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 获取Vosk模型（中文） + * 模型下载网盘：通过网盘分享的文件：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 更多模型下载地址：https://alphacephei.com/vosk/models + * @return + */ + public SpeechRecognizer geVoskRecognizer() { + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.VOSK); + /** + * 每个模型只支持一种语言，请下载对应语音的模型，模型下载地址：https://alphacephei.com/vosk/models + * 将模型解压后，将模型目录位置填写到此处 + */ + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/vosk-model-cn-0.22"); + /** + * macos m系列芯片需要手动下载依赖库，并指定位置，其他平台不需要 + * 下载地址：https://pan.baidu.com/s/1LZ_EX1XdTTp_f5ruud82MA?pwd=1234 提取码: 1234 + */ + config.setLibPath(Paths.get("/Users/wenjie/Downloads/vosk-arrch64-dylib-main/libvosk.dylib")); + return SpeechRecognizerFactory.getInstance().getModel(config); + } + + /** + * 获取Vosk模型（英文） + * 模型下载网盘：通过网盘分享的文件：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 更多模型下载地址：https://alphacephei.com/vosk/models + * @return + */ + public SpeechRecognizer geEnVoskRecognizer() { + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.VOSK); + /** + * 每个模型只支持一种语言，请下载对应语音的模型，模型下载地址：https://alphacephei.com/vosk/models + * 将模型解压后，将模型目录位置填写到此处 + */ + config.setModelPath("/Users/xxx/Documents/develop/model/speech/vosk-model-small-en-us-0.15"); +// config.setLibPath(Paths.get("/Users/xxx/Downloads/vosk-arrch64-dylib-main/libvosk.dylib")); + return SpeechRecognizerFactory.getInstance().getModel(config); + } + + + /** + * Vosk 语音识别 + * 支持 20 多种语言和方言——英语、印度英语、德语、法语、西班牙语、葡萄牙语、中文、俄语、土耳其语、越南语、意大利语、荷兰语、加泰罗尼亚语、阿拉伯语、希腊语、波斯语、菲律宾语、乌克兰语、哈萨克语、瑞典语、日语、世界语、印地语、捷克语、波兰语等 + * 注意事项: + * 1、每个模型只支持一种语言，请下载对应语言的模型 + * 2、如果音频中存在多种语言，不推荐使用vosk，可以使用Whisper + * 3、模型越大越准确 + * 4、暂不支持GPU使用，如需GPU使用需要自行编译：https://alphacephei.com/vosk/install + */ + @Test + public void testVosk() { + try { + SpeechRecognizer recognizer = geVoskRecognizer(); + //建议上传 WAV 格式音频。其他格式将自动转换为 WAV，可能影响处理速度 + R result = recognizer.recognize("src/main/resources/lff_zh.mp3"); + if (result.isSuccess()){ + log.info("识别成功:{}", JsonUtils.toJson(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Vosk 语音识别(使用Grammar语法规则) + * 支持 20 多种语言和方言——英语、印度英语、德语、法语、西班牙语、葡萄牙语、中文、俄语、土耳其语、越南语、意大利语、荷兰语、加泰罗尼亚语、阿拉伯语、希腊语、波斯语、菲律宾语、乌克兰语、哈萨克语、瑞典语、日语、世界语、印地语、捷克语、波兰语等 + * 注意事项: + * 1、每个模型只支持一种语言，请下载对应语言的模型 + * 2、如果音频中存在多种语言，不推荐使用vosk，可以使用Whisper + * 3、模型越大越准确 + * 4、暂不支持GPU使用，如需GPU使用需要自行编译：https://alphacephei.com/vosk/install + */ + @Test + public void testVoskWithGrammar() { + try { + //获取英文模型 + SpeechRecognizer recognizer = geEnVoskRecognizer(); + VoskParams voskParams = new VoskParams(); + //英文 + voskParams.setLanguage(Language.EN); + voskParams.setGrammar("[\"one two three four five six seven eight nine zero oh\"]"); + //建议上传 WAV 格式音频。其他格式将自动转换为 WAV，可能影响处理速度 + R result = recognizer.recognize("src/main/resources/test_en.wav",voskParams); + if (result.isSuccess()){ + log.info("识别成功:{}", JsonUtils.toJson(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Vosk 语音识别(使用Vosk内部识别器) + * 支持 20 多种语言和方言——英语、印度英语、德语、法语、西班牙语、葡萄牙语、中文、俄语、土耳其语、越南语、意大利语、荷兰语、加泰罗尼亚语、阿拉伯语、希腊语、波斯语、菲律宾语、乌克兰语、哈萨克语、瑞典语、日语、世界语、印地语、捷克语、波兰语等 + * 注意事项: + * 1、每个模型只支持一种语言，请下载对应语言的模型 + * 2、如果音频中存在多种语言，不推荐使用vosk，可以使用Whisper + * 3、模型越大越准确 + * 4、暂不支持GPU使用，如需GPU使用需要自行编译：https://alphacephei.com/vosk/install + */ + @Test + public void testVoskAdvanced() { + try { + VoskRecognizer recognizer = (VoskRecognizer)geVoskRecognizer(); + //使用vosk内部接口，需要指定识别音频的采样率 + Recognizer voskRecognizer = recognizer.createAdvancedRecognizer(16000); + voskRecognizer.setWords(true); + voskRecognizer.setPartialWords(true); + // 使用vosk内部接口，只支持wav格式 + String audioPath = "src/main/resources/lff_zh.wav"; + InputStream ais = AudioSystem.getAudioInputStream(new BufferedInputStream(new FileInputStream(audioPath))); + int nbytes; + byte[] b = new byte[4096]; + while ((nbytes = ais.read(b)) >= 0) { + if (voskRecognizer.acceptWaveForm(b, nbytes)) { + log.info(voskRecognizer.getResult()); + } else { + log.info(voskRecognizer.getPartialResult()); + } + } + log.info(voskRecognizer.getFinalResult()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 实时语音识别（VOSK） + */ + @Test + public void testVoskRealTime() { + try { + VoskRecognizer recognizer = (VoskRecognizer)geVoskRecognizer(); + //使用vosk内部接口，需要指定识别音频的采样率 + Recognizer voskRecognizer = recognizer.createAdvancedRecognizer(16000); + voskRecognizer.setWords(true); + voskRecognizer.setPartialWords(true); + // 设置音频格式: 16kHz, 16bit, 单声道, PCM + AudioFormat format = new AudioFormat(16000, 16, 1, true, false); + // 获取可用的 TargetDataLine + DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); + if (!AudioSystem.isLineSupported(info)) { + System.out.println("麦克风不支持该格式"); + System.exit(0); + } + TargetDataLine microphone = (TargetDataLine) AudioSystem.getLine(info); + microphone.open(format); + microphone.start(); + int nbytes; + byte[] b = new byte[4096]; + while ((nbytes = microphone.read(b,0,b.length)) >= 0) { + if (voskRecognizer.acceptWaveForm(b, nbytes)) { + log.info(voskRecognizer.getResult()); + } else { + log.info(voskRecognizer.getPartialResult()); + } + } + log.info(voskRecognizer.getFinalResult()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + +} diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java new file mode 100644 index 0000000..04f68fe --- /dev/null +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java @@ -0,0 +1,204 @@ +package smartai.examples.speech.tts; + + +import ai.djl.modality.audio.Audio; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.speech.tts.config.TtsModelConfig; +import cn.smartjavaai.speech.tts.entity.SherpaTtsParams; +import cn.smartjavaai.speech.tts.enums.TtsModelEnum; +import cn.smartjavaai.speech.tts.factory.TtsModelFactory; +import cn.smartjavaai.speech.tts.model.SherpaTtsModel; +import cn.smartjavaai.speech.tts.model.TtsModel; +import cn.smartjavaai.speech.utils.AudioUtils; +import com.k2fsa.sherpa.onnx.GeneratedAudio; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * 语音合成demo（TTS） + * sherpa-onnx模型及依赖库下载链接: + * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases + * 2、（推荐）TTS模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + * 3、百度网盘下载：https://pan.baidu.com/s/186REUf7p1z0HH9AZNnwCUg?pwd=1234 提取码: 1234 + * @author dwj + * @date 2025/10/22 + */ +@Slf4j +public class TtsDemo { + + public static String enText = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + public static String zhText = "有问题，请拨打110或者手机18601239876。我们的价值观是真诚热爱！"; + + String znEnText = + "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki." + + " 你觉得中英文说的如何呢？"; + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + /** + * 获取Vits中文模型 + * @return + */ + public TtsModel getVitsZhModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_VITS); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/vits-zh-hf-keqing"); + modelConfig.setModelName("keqing.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 1); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Matcha中文模型 + * @return + */ + public TtsModel getMatchaZhModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker"); + modelConfig.setModelName("model-steps-3.onnx"); + modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 1); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Kokoro多语言模型 + * @return + */ + public TtsModel getKokoroZhEnModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_KOKORO); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/kokoro-multi-lang-v1_0"); + modelConfig.setModelName("model.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 2); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Matcha模型（英文） + * @return + */ + public TtsModel getMatchaEnModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-en_US-ljspeech"); + modelConfig.setModelName("model-steps-3.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx"); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 2); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 语音合成Vits（中文） + */ + @Test + public void testVitsZhTts() throws IOException { + TtsModel ttsModel = getVitsZhModel(); + SherpaTtsParams params = new SherpaTtsParams(); + //语速 + params.setSpeed(1f); + //说话人ID + params.setSpeakerId(100); + //生成音频到本地路径 + ttsModel.generate(zhText, params, "/Users/wenjie/Downloads/tts-vits-zh.wav"); + } + + /** + * 语音合成Matcha（中文） + */ + @Test + public void testMatchaZhTts() throws IOException { + TtsModel ttsModel = getMatchaZhModel(); + SherpaTtsParams params = new SherpaTtsParams(); + //语速 + params.setSpeed(1f); + //说话人ID + params.setSpeakerId(0); +// ttsModel.generate(zhText, params,"/Users/wenjie/Downloads/tts-matcha-zh.wav"); + R