diff --git a/README.md b/README.md index 5d11bca..a5123c1 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸检测

+

人脸检测(Face Detection)

- 5点人脸关键点定位
@@ -59,7 +59,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸比对1：1

+

人脸比对1：1
(Face Recognition)

- 人脸对齐
@@ -72,7 +72,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人证核验

+

人证核验
(Face ID Verification)

@@ -84,7 +84,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸比对1：N

+

人脸比对1：N
(Face Recognition)

- 人脸对齐
- 人脸注册
- 人脸库查询
@@ -101,11 +101,11 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的

人脸属性检测

- - 性别检测
- - 年龄检测
- - 口罩检测
- - 眼睛状态检测
- - 脸部姿态检测
+ - 性别检测(GenderDetection)
+ - 年龄检测(AgeDetection)
+ - 口罩检测(Face Mask Detection)
+ - 眼睛状态检测(EyeClosenessDetection)
+ - 脸部姿态检测(FacePoseEstimation)
@@ -117,7 +117,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

活体检测

+

活体检测
(FaceLivenessDetection)

- 图片和视频活体检测
@@ -130,7 +130,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

人脸表情识别

+

人脸表情识别
(FacialExpressionRecognition)

- 7种表情检测
@@ -143,7 +143,8 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

目标检测

+

目标检测(Object Detection)

+ - 视频流目标检测：rtsp、摄像头、视频文件等
@@ -151,11 +152,40 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
+ +
+ +
+ - +
-

自定义目标训练+检测

+

语义分割
(Semantic Segmentation)

+
+ + +
+ +
+ + + + +
+

实例分割
(Instance Segmentation)

+
+ + +
+ +
+ + + + +
+

自定义目标训练+检测
(Object Detection)

@@ -167,7 +197,55 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

OCR文字识别

+

行人检测(Person Detection)

+
+ + +
+ +
+ + + + +
+

人类动作识别
(Action Recognition)

+
+ + +
+ +
+ + + + +
+

OBB旋转框检测
(Oriented Bounding Boxes Object Detection)

+
+ + +
+ +
+ + + + +
+

姿态估计
(Pose Estimation)

+
+ + +
+ +
+ + + + +
+

OCR文字识别
(Text Recognition)

- 支持任意角度文字识别
- 支持印刷体识别
- 支持手写字识别
@@ -183,7 +261,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的

OCR文字识别

- - 表格识别
+ - 表格识别
(Table Structure Recognition)
@@ -195,7 +273,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

车牌识别

+

车牌识别
(License Plate Recognition)

- 单层/双层检测
- 车牌颜色识别
@@ -214,7 +292,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

机器翻译

+

机器翻译
(Machine Translation)

- 200多种语言互相翻译
@@ -227,7 +305,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
-

语音识别

+

语音识别
(Speech Recognition)

- 支持100种语言
- 支持实时语音识别
@@ -243,13 +321,13 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 -## 🚀 核心亮点 +## 🚀 SmartJavaAI 解决了什么问题？ 针对 Java 开发者在集成智能算法时常见的两大痛点： - 🐍 主流AI深度学习框架（PyTorch/TensorFlow）的Python生态与Java工程体系割裂 -- ⚙️ 现有算法方案分散杂乱，封装不统一，使用门槛高，不易直接服务于 Java 业务开发 +- ⚙️ 现有AI集成方案分散杂乱，封装不统一，使用门槛高，不易直接服务于 Java 业务开发 我们实现了： @@ -257,7 +335,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ✅ **支持多种深度学习引擎** - Pytorch、Tensorflow、MXNet、ONNX Runtime -✅ **功能丰富** - 当前支持人脸识别与目标检测，未来将陆续支持 OCR、图像分类、NLP 等多个 AI 领域任务，构建全面的智能算法体系。 +✅ **功能丰富** - 当前支持人脸识别、目标检测、OCR、NLP 等多个 AI 领域任务，构建全面的智能算法体系。 ✅ **跨平台兼容** - 支持Windows/Linux/macOS系统（x86 & ARM架构） @@ -265,11 +343,8 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 📌 支持功能 - -### ✅ 已实现功能 - - **人脸识别** - - 人脸检测：5点人脸关键点定位 + - 人脸检测：人脸区域检测、5点人脸关键点定位 - 人脸识别：人脸512维特征提取、人脸对齐、1:1 人脸比对、1:N 人脸识别 - 人脸库：人脸注册、更新、查询、删除(支持向量数据库milvus/sqlite) - 人脸属性检测：性别、年龄、口罩、眼睛状态、脸部姿态 @@ -277,8 +352,19 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 人脸表情识别：7种表情识别 - 人脸质量评估：亮度评估、清晰度评估、完整度评估、姿态评估、遮挡评估 - **目标检测** - - 支持多种主流模型：兼容 YOLOv3、YOLOv5、YOLOv8、YOLOv11、YOLOv12、SSD 等目标检测算法 + - 支持多种主流模型：集成 YOLOv5、YOLOv8、YOLOv11、YOLOv12、Tensorflow Object Detection 等目标检测算法 - 支持自定义模型加载：可无缝加载并部署用户自行训练的目标检测模型 + - 集成行人检测模型 +- **语义分割** + - 集成DeepLabV3模型 +- **实例分割** + - 集成YOLOv8-seg、YOLOv11-seg、Mask R-CNN等模型 +- **OBB旋转框目标检测** + - 集成YOLOv11-obb模型 +- **动作识别** + - 支持KINETICS400数据集中400个人类动作识别 +- **姿态估计** + - 集成YOLOv8-pose、YOLOv11-pose等模型 - **OCR文字识别** - 支持PaddleOCR 3.0模型：集成最新PP-OCRv5、PP-OCRv4、表格结构识别模型(SLANet_plus)、文本行方向分类模型 - 支持任意角度识别，方向校准 @@ -294,29 +380,29 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 🌟 AI集成方式对比 -| 方案 | 技术特点 | 优点 | 缺点 | -|---------------------|--------------------------------------------------------------------------|---------------------------------------------------------------------|-----------------------------------------------| -| **OpenCV** | 传统图像处理方案 | ✅ 提供java接口
✅ 轻量级部署
✅ 社区资源丰富 | ❌ 基于传统算法精度低(60%-75%)
❌ 需本地安装环境 | +| 方案 | 技术特点 | 优点 | 缺点 | +|---------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------|-----------------------------------------------| +| **OpenCV** | 传统图像处理方案 | ✅ 提供java接口
✅ 轻量级部署
✅ 社区资源丰富 | ❌ 基于传统算法精度低(60%-75%)
❌ 需本地安装环境 | | ‌**商业闭源SDK（如虹软等）**‌ | 商业级闭源解决方案 | ✅ 开箱即用
✅ 提供完整文档和SDK
✅ 支持离线活体检测 | ❌ 免费版需年度授权更新
❌ 商业授权费用高
❌ 代码不可控 | | ‌**云API(阿里云)**‌ | SaaS化云端服务 | ✅ 零部署成本
✅ 支持高并发
✅ 自带模型迭代 | ❌ 网络延迟风险(200-800ms)
❌ 按调用量计费
❌ 有数据安全风险 | | ‌**Python混合调用**‌ | 跨语言调用方案 | ✅ 可集成PyTorch/TF等框架
✅ 支持自定义算法
✅ 识别精度高 | ❌ 需维护双语言环境
❌ 进程通信性能损耗(30%+)
❌ 异常处理复杂度翻倍 | -| ‌**JNI/JNA**‌ | 跨语言底层调用方案 | ✅ 直接调用 C/C++ 高性能算法库✅ 支持调用各种原生成熟库✅ 可封装成通用工具Jar | ❌ 开发成本高，JNI更复杂❌ 跨平台兼容性差 | +| ‌**JNI/JNA**‌ | 跨语言底层调用方案 | ✅ 直接调用 C/C++ 高性能算法库
✅ 支持调用各种原生成熟库
✅ 可封装成通用工具Jar | ❌ 开发成本高，JNI更复杂
❌ 跨平台兼容性差 | | ‌**DJL框架**‌ | 深度学习框架 | ✅ 纯Java实现
✅ 支持主流深度学习框架
✅ 可加载预训练模型(99%+) | ❌ 需掌握DL知识
❌ 需处理模型加载、预处理、后处理等复杂技术细节 | -| ‌**SmartJavaAI**‌ | java深度学习工具包 | ✅ 支持主流深度学习框架
✅ 提供丰富、开箱即用API
✅ 上手简单，单一Jar包集成 | 无 | +| ‌**SmartJavaAI**‌ | java深度学习工具包 | ✅ 支持主流深度学习框架
✅ 提供丰富、开箱即用API
✅ 上手简单，单一Jar包集成 | 无 | ## 🛠️包含组件 -| 模块 | 介绍 | -|-----------------------------|------------------------------| -| smartjavaai-common | 基础通用模块，封装了公共功能，供各算法模块共享使用 | -| smartjavaai-bom | 依赖管理模块 | -| smartjavaai-face | 人脸功能模块 | -| smartjavaai-objectdetection | 目标检测模块 | -| smartjavaai-ocr | OCR文字识别模块 | -| smartjavaai-translate | 机器翻译模块 | -| smartjavaai-speech | 语音功能模块，包含 ASR 和 TTS | +| 模块 | 介绍 | +|-----------------------|---------------------------| +| common | 基础通用模块，封装了公共功能，供各算法模块共享使用 | +| bom | 依赖管理模块 | +| face | 人脸功能模块 | +| vision | 通用视觉模块(目标检测等功能) | +| ocr | OCR文字识别模块 | +| translate | 机器翻译模块 | +| speech | 语音功能模块，包含 ASR 和 TTS | 可以根据需求对每个模块单独引入，也可以通过引入`smartjavaai-all`方式引入所有模块。 @@ -335,10 +421,14 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ### 1、环境要求 - Java 版本：**JDK 8或更高版本** -- 操作系统：不同模型支持的系统不一样，具体请查看文档 +- 操作系统：不同模型支持的系统不一样，具体请查看[文档](http://doc.smartjavaai.cn) ### 2、Maven -在项目的pom.xml的dependencies中加入以下内容（全部功能），也可以根据需求对每个模块单独引入: + +在项目的 `pom.xml` 的 `dependencies` 中可以一次性引入全部功能（如下所示）。 +⚠️ **注意：不推荐直接引入全部依赖**，更推荐根据实际需求，按功能模块单独引入，避免引入不必要的包。 + +详细引入方式请查看 [文档](http://doc.smartjavaai.cn/Install)、或查看[示例代码](https://gitee.com/dengwenjie/SmartJavaAI/tree/master/examples) ```xml @@ -347,6 +437,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 1.0.24 ``` + ### 3、完整示例代码 [示例代码](https://gitee.com/dengwenjie/SmartJavaAI/tree/master/examples) @@ -363,39 +454,33 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 **人脸检测模型(FaceDetection、FaceLandmarkExtraction)** -支持功能： -- 人脸检测 -- 5点人脸关键点定位 - -| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | -| ----------- |---------|-------------------------|----------------------------------------------------------------------------------------| -| RetinaFace | PyTorch | 高精度人脸检测模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | -| UltraLightFastGenericFace | PyTorch | 针对边缘计算设备设计的轻量人脸检测模型 | [Github](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | -| SeetaFace6 | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|------------------------------|-------------|------------------------------|---------------------------------------------------------------------------------| +| MTCNN(均衡) | PyTorch | 使用最广泛的模型之一，经典多阶段人脸检测，速度与精度均衡 | [Github](https://github.com/timesler/facenet-pytorch) | +| SeetaFace6(均衡模型) | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| RetinaFace(高精度) | PyTorch | 基于单阶段检测的多尺度人脸检测 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| RetinaFace_1080x720(高精度) | OnnxRuntime | 针对高分辨率图像优化的 RetinaFace 模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| RetinaFace_640x640(高精度) | OnnxRuntime | 针对低分辨率图像优化的 RetinaFace 模型 | [Github](https://github.com/biubug6/Pytorch_Retinaface) | +| yolov5face_n_0.5_320x320(极速) | OnnxRuntime | YOLOv5 小型人脸检测模型，设计轻量，适合快速推理 | 无 | +| yolov5face_m_640x640(极速) | OnnxRuntime | YOLOv5 中型人脸检测模型 | 无 | +| UltraLightFastGenericFace(极速) | PyTorch | 针对边缘计算设备设计的轻量人脸检测模型 | [Github](https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB) | --- **人脸识别模型(FaceRecognition)** -支持功能： -- 人脸512维特征提取 -- 人脸对齐(人脸矫正) -- 人脸特征比对（内积[IP]、欧氏距离[L2]、余弦相似度[COSINE]） -| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | -|-------|---------|---------------------------------------------------------|----------------------------------------------------------------------------------------| -| InsightFace_IR-SE50| PyTorch | （高精度）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | -| InsightFace_Mobilefacenet | PyTorch | （轻量级）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | -| FaceNet | PyTorch | 基于 PyTorch 的 Inception ResNet（V1）模型仓库 | [Github](https://github.com/timesler/facenet-pytorch) | -| ElasticFace| PyTorch | 基于 CVPRW2022 论文《ElasticFace: Elastic Margin Loss for Deep Face Recognition》实现的人脸识别模型| [Github](https://github.com/fdbtrs/ElasticFace) | -| SeetaFace6 | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------------------------|---------|--------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| +| InsightFace_IR-SE50(高精度) | PyTorch | 这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | +| InsightFace_Mobilefacenet(极速) | PyTorch | （轻量级）这是对 ArcFace（论文）和 InsightFace（GitHub）的 PyTorch 重新实现 | [Github](https://github.com/TreB1eN/InsightFace_Pytorch) | +| FaceNet(均衡) | PyTorch | 基于 PyTorch 的 Inception ResNet（V1）模型仓库 | [Github](https://github.com/timesler/facenet-pytorch) | +| ElasticFace(高精度) | PyTorch | 基于 CVPRW2022 论文《ElasticFace: Elastic Margin Loss for Deep Face Recognition》实现的人脸识别模型 | [Github](https://github.com/fdbtrs/ElasticFace) | +| SeetaFace6(均衡) | C++ | 中科视拓最新开放的开源免费的全栈人脸识别工具包 | [Github](https://github.com/seetafaceengine/SeetaFace6) | **静态活体检测(RGB)模型（Silent face-anti-spoofing、FaceLivenessDetection）** -支持功能： -- 检测图片中的人脸是否为来自认证设备端的近距离裸拍活体人脸对象(裸拍活体正面人脸是指真人未经重度PS、风格化、人工合成等后处理的含正面人脸) - | 模型名称 | 引擎 | 模型简介 | 模型开源网站 | |-----------|----------------|-------------------------|----------------------------------------------------------------------| | MiniVision| OnnxRuntime | 小视科技的静默活体检测 | [Github](https://github.com/minivision-ai/Silent-Face-Anti-Spoofing) | @@ -446,21 +531,23 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 #### 目标检测模型 -支持功能： -- 自训练模型推理 -- yolov3~yolov12 系列 - **YOLO 系列** -|模型名称 | 引擎 | 版本 | 大小（Backbone） |
数据集
|
精度
|
速度
|
适用场景
| -| :---------- |-------------| -------| ----------- |----------------------------------------|-----------------------------------|-------|--------------------------------------| -| YOLO12N | OnnxRuntime | v12 | 极轻量 | COCO | 高 | 极快 | YOLO 系列最新版本，精度与速度进一步优化，适合高实时性要求场景 | -|YOLO11N | PyTorch | v11 | 极轻量 | COCO | 中等偏上 | 极快 | 与 v8n 类似，版本更新点在兼容性与 API | -|YOLOV8N | PyTorch | v8 | 极轻量 | COCO | 中等偏上 | 极快 | 对实时性要求极高的应用 | -|YOLOV5S | PyTorch | v5 | 小型 | COCO | 较高 | 非常快 | 常见通用场景，算力资源有限时优选 | -|YOLOV5S_ONNXRUNTIME| OnnxRuntime | v5 | 小型 | COCO | 较高 | 加速（需 ONNX 支持） | Windows/Linux 通用加速部署 | -|YOLO (MXNet / 通用模型) | MXNet | v3 | DarkNet‑53 | COCO | 较高 | 快 | 需要 MXNet 生态或复现老项目时使用 | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|----------------|-------------------------|----------------------------------------------------------------------| +| YOLOV12 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV11 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV8 | OnnxRuntime | 最流行的目标检测模型 | [Github](https://docs.ultralytics.com/zh/models/) | +**Tensorflow系列** + +仅测试了以下Tensorflow目标检测模型 + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|-------------|----------------|----------------------------------------------------------------------| +| EfficientDet | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | +| SSD MobileNet V2 | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | +| Faster RCNN Inception Resnet V2 | Tensorflow | Tensorflow目标检测 | [Github](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) | **SSD 系列** @@ -472,21 +559,59 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 |SSD_300_VGG16_ATROUS_VOC| MXNet | VGG‑16 | 300×300 | Pascal VOC | 中等偏上 | 快 | VOC 数据集同类任务；资源受限时使用| |SSD_512_MOBILENET1_VOC| MXNet | MobileNet‑1.0 | 512×512 | Pascal VOC | 中等 | 快 | 嵌入式/移动端设备；算力和内存都很有限| +--- -**YOLOv3 变体系列** +#### 语义分割模型 -|模型名称| 引擎 | 骨干网络 |
数据集
| 输入尺寸 |
精度
|
速度
|
适用场景
| -| :-----|---------- | -------| ----------- |------------| ---------|-------|--------------------------------------| -|YOLO3_DARKNET_VOC_416|MXNet | DarkNet‑53 | VOC | 416×416 | 高 | 中等 | VOC 任务复现；精度优先 | -|YOLO3_DARKNET_COCO_320 |MXNet| DarkNet‑53 | COCO | 320×320 | 中等 | 快 | COCO 小模型测试；资源受限 | -|YOLO3_DARKNET_COCO_416 |MXNet| DarkNet‑53 | COCO | 416×416 | 高 | 中等 | 通用 COCO 部署；精度优先 | -|YOLO3_DARKNET_COCO_608 |MXNet| DarkNet‑53 | COCO | 608×608 | 很高 | 慢| 批量离线推理；精度要求极高 | -|YOLO3_MOBILENET_VOC_320 |MXNet| MobileNet‑V1 | VOC | 320×320 | 中等 | 非常快| 嵌入式设备；VOC 小目标任务 | -|YOLO3_MOBILENET_VOC_416 |MXNet| MobileNet‑V1 | VOC | 416×416 | 高 | 快| 移动端 VOC 部署 | -|YOLO3_MOBILENET_COCO_320 |MXNet| MobileNet‑V1 | COCO | 320×320 | 中等 | 非常快 | 嵌入式设备；COCO 小目标任务 | -|YOLO3_MOBILENET_COCO_416 |MXNet| MobileNet‑V1 | COCO | 416×416 | 高 | 快 | 移动端 COCO 部署 | -|YOLO3_MOBILENET_COCO_608 |MXNet| MobileNet‑V1 | COCO | 608×608 | 很高 | 中等 | 对精度要求较高的移动端任务 | +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|---------|---------|-------------------------|--------| +| DeepLabv3 | PyTorch | DeepLabv3 是一种图像分割模型，可以把图片里的每个像素分类出来 | 无 | +--- + +#### 实例分割模型 + + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|---------|------------------------------------|------------| +| YOLOV8-SEG | OnnxRuntime | Ultralytics在COCO 数据集 上训练的模型 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +| YOLOV11-SEG | OnnxRuntime | Ultralytics在COCO 数据集 上训练的模型 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +| Mask R-CNN | MXNet | Mask R-CNN 是一种在目标检测基础上，同时为每个物体生成像素级分割区域的深度学习模型 | 无 | +--- + +#### OBB旋转框目标检测模型 + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|---------|--------------------------------|------------| +| YOLOV11-OBB | OnnxRuntime | Ultralytics在DOTAv1 数据集 上训练的模型、通过引入一个额外的角度来更准确地定位图像中的对象 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | +--- + +#### 行人检测模型 + +| 模型名称 | 引擎 | 模型开源网站 | +|-------------|---------|-----| +| YOLOV8_PERSON | OnnxRuntime | [Github](https://github.com/YapaLab/yolo-face) | + +--- + +#### 人类动作识别模型 + +| 模型名称 | 引擎 | +|-------------|---------| +| VIT_BASE_PATCH16 | PyTorch | +| INCEPTIONV3_KINETICS400 | OnnxRuntime | +| INCEPTIONV1_KINETICS400 | OnnxRuntime | +| RESNET_V1B_KINETICS400 | OnnxRuntime | +--- + +#### 姿态估计模型 + + +| 模型名称 | 引擎 | 模型开源网站 | +|--------------|-------------|----------------------------------------------------------| +| YOLO11N-POSE | OnnxRuntime | [Github](https://docs.ultralytics.com/zh/tasks/pose/) | +| YOLO8N-POSE | OnnxRuntime | [Github](https://docs.ultralytics.com/zh/tasks/pose/) | +| SIMPLE_POSE | MXNet | 无 | --- #### OCR 模型 @@ -555,9 +680,11 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 支持功能 - 200多语言互相翻译 -| 模型名称 | 模型简介 | 模型官网 | -| ------------------------------ | ------------------------ |-----------------------------------------------| -| NLLB-200 | Meta AI 开发的一个先进的单一多语言机器翻译模型 | [Github](https://github.com/facebookresearch/fairseq/tree/nllb) | +| 模型名称 | 模型简介 | 模型官网 | +|---------------|-----------------------------------------|-----------------------------------------------| +| NLLB-200 | Meta AI 开发的一个先进的单一多语言机器翻译模型 | [Github](https://github.com/facebookresearch/fairseq/tree/nllb) | +| OPUS_MT_ZH_EN | Helsinki-NLP团队开发的中英翻译模型，基于Transformer架构 | [huggingface](https://huggingface.co/Helsinki-NLP/opus-mt-zh-en) | +| OPUS_MT_EN_ZH | Helsinki-NLP团队开发的英中翻译模型，基于Transformer架构 | [huggingface](https://huggingface.co/Helsinki-NLP/opus-mt-en-zh) | --- @@ -606,7 +733,19 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 近期更新日志 -## [v1.0.24] - 2025-08-09 +## [v1.0.24] - 2025-09-07 +- 【人脸检测】新增6个模型(MTCNN、YOLOV5、RetinaFace小尺寸版)，大幅提升性能 +- 【人脸识别】新增Seetaface6轻量模型 +- 【目标检测】支持视频流目标检测（rtsp、视频文件等） +- 【目标检测】支持tensorflow2目标检测模型 +- 【目标检测】新增行人检测模型(yolo-person) +- 【通用视觉】新增4个动作识别模型 +- 【通用视觉】新增语义分割模型 +- 【通用视觉】新增5个实例分割模型(含yolov8-seg、yolov11-seg) +- 【通用视觉】新增yolo-obb11旋转框检测(含yolov11-obb) +- 【通用视觉】新增5个姿态估计模型(含yolov8-pose、yolov11-pose) + +## [v1.0.23] - 2025-08-09 - 新增 语音识别模块，集成 OpenAI 开源的 Whisper 和 Vosk - 修复 质量评估模型的 Bug - 修复 OCR 模块 recognizeAndDraw 方法的 Bug @@ -628,17 +767,3 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 人脸识别：支持ID查询及分页获取人脸信息 - 活体检测：视频检测支持设置最大帧数 -## [v1.0.19] - 2025-07-06 -- 人脸模块：新增小视科技（MiniVision）活体检测模型 -- 人脸模块：新增阿里通义工作室活体检测模型 -- 人脸模块：新增 2 个表情识别模型 -- 人脸模块：新增 InsightFace 和 ElasticFace 人脸识别模型 -- 人脸模块：新增 Seetaface6 质量评估模型 -- 目标检测模块：支持更多自定义模型参数配置 -- 人脸模块：支持 Base64 编码图片输入 -- 通用功能：实现 AutoCloseable 接口，支持资源自动释放 -- OCR 模块：修复加方向矫正后无法连续识别的问题 -- 人脸模块：修复人脸更新后的缓存异常问题 -- 其他：优化部分功能与细节体验 - - diff --git a/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java b/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java index a0df61e..9aaa0f7 100644 --- a/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java +++ b/common/src/main/java/cn/smartjavaai/common/entity/DetectionResponse.java @@ -8,14 +8,13 @@ import java.util.List; /** * 检测结果 * @author dwj - * @date 2025/4/12 */ @Data public class DetectionResponse { private List detectionInfoList; - private Image drawnImage; + private transient Image drawnImage; public DetectionResponse() { diff --git a/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java b/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java index 7490df3..2836971 100644 --- a/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java +++ b/common/src/main/java/cn/smartjavaai/common/entity/ObbDetInfo.java @@ -1,11 +1,14 @@ package cn.smartjavaai.common.entity; +import lombok.Data; + import java.util.List; /** * 定向边界框 检测结果 * @author dwj */ +@Data public class ObbDetInfo { /** diff --git a/common/src/main/java/cn/smartjavaai/common/utils/FrameConverterUtil.java b/common/src/main/java/cn/smartjavaai/common/utils/FrameConverterUtil.java deleted file mode 100644 index b2ac439..0000000 --- a/common/src/main/java/cn/smartjavaai/common/utils/FrameConverterUtil.java +++ /dev/null @@ -1,66 +0,0 @@ -package cn.smartjavaai.common.utils; - -import ai.djl.modality.cv.Image; -import ai.djl.modality.cv.ImageFactory; -import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacv.OpenCVFrameConverter; -import org.bytedeco.opencv.opencv_core.CvMat; -import org.bytedeco.opencv.opencv_core.Mat; -import org.opencv.core.CvType; - -import java.awt.image.BufferedImage; - -/** - * @author dwj - * @date 2025/8/27 - */ -public class FrameConverterUtil { - - /** - * 将 Bytedeco Mat 转为 DJL Image - * 支持 1/3/4 通道 - */ - public static Image matToDJLImage(Mat cvMat) { - if (cvMat == null || cvMat.empty()) { - return null; - } - - int width = cvMat.cols(); - int height = cvMat.rows(); - int channels = cvMat.channels(); - - int[] pixels = new int[width * height]; - - if (channels == 1) { // 灰度图 - byte[] data = new byte[width * height]; - cvMat.data().get(data); - for (int i = 0; i < width * height; i++) { - int gray = data[i] & 0xFF; - pixels[i] = (0xFF << 24) | (gray << 16) | (gray << 8) | gray; - } - } else if (channels == 3) { // BGR - byte[] data = new byte[width * height * 3]; - cvMat.data().get(data); - for (int i = 0; i < width * height; i++) { - int b = data[i * 3] & 0xFF; - int g = data[i * 3 + 1] & 0xFF; - int r = data[i * 3 + 2] & 0xFF; - pixels[i] = (0xFF << 24) | (r << 16) | (g << 8) | b; - } - } else if (channels == 4) { // BGRA - byte[] data = new byte[width * height * 4]; - cvMat.data().get(data); - for (int i = 0; i < width * height; i++) { - int b = data[i * 4] & 0xFF; - int g = data[i * 4 + 1] & 0xFF; - int r = data[i * 4 + 2] & 0xFF; - int a = data[i * 4 + 3] & 0xFF; - pixels[i] = (a << 24) | (r << 16) | (g << 8) | b; - } - } else { - throw new IllegalArgumentException("只支持 1/3/4 通道图像"); - } - - return ImageFactory.getInstance().fromPixels(pixels, width, height); - } -} diff --git a/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java index e0b6344..e925453 100644 --- a/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java @@ -3,8 +3,11 @@ package cn.smartjavaai.common.utils; import ai.djl.modality.cv.BufferedImageFactory; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.CategoryMask; import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.ndarray.NDArray; +import ai.djl.util.RandomUtils; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; import org.opencv.core.Mat; @@ -17,6 +20,7 @@ import java.awt.*; import java.awt.image.BufferedImage; //import java.awt.image.ColorConvertOp; import java.awt.image.ComponentSampleModel; +import java.awt.image.DataBufferByte; import java.awt.image.ImageObserver; import java.io.File; import java.io.IOException; @@ -500,6 +504,86 @@ public class ImageUtils { } + /** + * 拷贝图片 + * @param src + * @return + */ + public static BufferedImage copyBufferedImage(BufferedImage src) { + BufferedImage copy = new BufferedImage(src.getWidth(), src.getHeight(), src.getType()); + Graphics2D g = copy.createGraphics(); + g.drawImage(src, 0, 0, null); + g.dispose(); + return copy; + } + + + /** + * 拷贝图片 + * @param src + * @return + */ + public static Image copy(Image src) { + Object srcData = src.getWrappedImage(); + //当图片未BufferedImage，DJL的duplicate会有问题 + if (srcData instanceof BufferedImage) { + return SmartImageFactory.getInstance().fromImage(copyBufferedImage((BufferedImage) srcData)); + }else{ + return src.duplicate(); + } + } + + /** + * 为不同分类生成不同颜色 + * @param background + * @param opacity + * @param classes + * @return + */ + public static int[] generateColors(int background, int opacity, List classes) { + int[] colors = new int[classes.size()]; + colors[0] = background; + for (int i = 1; i < classes.size(); i++) { + int red = RandomUtils.nextInt(256); + int green = RandomUtils.nextInt(256); + int blue = RandomUtils.nextInt(256); + colors[i] = opacity << 24 | red << 16 | green << 8 | blue; + } + return colors; + } + + /** + * 生成不同颜色遮罩 + * @param colors + * @param mask + * @return + */ + public static Image getColorOverlay(int[] colors,int[][] mask) { + int height = mask.length; + int width = mask[0].length; + int[] pixels = new int[width * height]; + for (int h = 0; h < height; h++) { + for (int w = 0; w < width; w++) { + int index = mask[h][w]; + pixels[h * width + w] = colors[index]; + } + } + return SmartImageFactory.getInstance().fromPixels(pixels, width, height); + } + + /** + * 绘制遮罩 + * @param categoryMask + * @param image + * @param opacity + * @param background + */ + public static void drawMask(CategoryMask categoryMask, Image image, int opacity, int background) { + int[] colors = generateColors(background, opacity, categoryMask.getClasses()); + Image maskImage = getColorOverlay(colors, categoryMask.getMask()); + image.drawImage(maskImage, true); + } + diff --git a/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java index ea782d1..909cf6a 100644 --- a/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java @@ -239,27 +239,27 @@ public class OpenCVUtils { * @param src Bytedeco Mat (BGR 或 BGRA) * @return OpenCV Mat (BGR 或 BGRA) */ - public static org.opencv.core.Mat convertToOpenCVMat(org.bytedeco.opencv.opencv_core.Mat bMat) { - - - try { - int width = bMat.cols(); - int height = bMat.rows(); - int channels = bMat.channels(); - - // 创建 OpenCV Mat - org.opencv.core.Mat cvMat = new org.opencv.core.Mat(height, width, channels == 3 ? CvType.CV_8UC3 : CvType.CV_8UC1); - - // 从 bytedeco Mat 获取像素数据 - byte[] data = new byte[width * height * channels]; - bMat.data().get(data); - - // 填充到 OpenCV Mat - cvMat.put(0, 0, data); - return cvMat; - } catch (Throwable e) { - e.printStackTrace(); - } - return null; - } +// public static org.opencv.core.Mat convertToOpenCVMat(org.bytedeco.opencv.opencv_core.Mat bMat) { +// +// +// try { +// int width = bMat.cols(); +// int height = bMat.rows(); +// int channels = bMat.channels(); +// +// // 创建 OpenCV Mat +// org.opencv.core.Mat cvMat = new org.opencv.core.Mat(height, width, channels == 3 ? CvType.CV_8UC3 : CvType.CV_8UC1); +// +// // 从 bytedeco Mat 获取像素数据 +// byte[] data = new byte[width * height * channels]; +// bMat.data().get(data); +// +// // 填充到 OpenCV Mat +// cvMat.put(0, 0, data); +// return cvMat; +// } catch (Throwable e) { +// e.printStackTrace(); +// } +// return null; +// } } diff --git a/common/src/main/java/cn/smartjavaai/common/utils/VideoUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/VideoUtils.java deleted file mode 100644 index d43ddda..0000000 --- a/common/src/main/java/cn/smartjavaai/common/utils/VideoUtils.java +++ /dev/null @@ -1,77 +0,0 @@ -package cn.smartjavaai.common.utils; - -import org.bytedeco.ffmpeg.global.avcodec; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.FFmpegFrameRecorder; -import org.bytedeco.javacv.Frame; -import org.bytedeco.javacv.OpenCVFrameConverter; -import org.bytedeco.opencv.global.opencv_core; -import org.bytedeco.opencv.opencv_core.Mat; - -/** - * 视频工具类 - * @author dwj - * @date 2025/7/17 - */ -public class VideoUtils { - - /** - * 视频旋转 - * @param inputPath 输入视频路径 - * @param outputPath 输出视频路径 - * @param angle 旋转角度 - * @param format 视频格式 - * @param videoCodec 视频编码器 - * @throws FFmpegFrameRecorder.Exception - * @throws FFmpegFrameGrabber.Exception - */ - public static void rotateVideo(String inputPath, String outputPath, int angle, String format, int videoCodec) throws FFmpegFrameRecorder.Exception, FFmpegFrameGrabber.Exception { - FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(inputPath); - grabber.start(); - int inputWidth = grabber.getImageWidth(); - int inputHeight = grabber.getImageHeight(); - int outputWidth = inputWidth; - int outputHeight = inputHeight; - - if (angle == 90 || angle == 270) { - outputWidth = inputHeight; - outputHeight = inputWidth; - } - FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(outputPath, - outputWidth, outputHeight, grabber.getAudioChannels()); - recorder.setVideoCodec(videoCodec); - recorder.setFormat(format); - recorder.start(); - Frame frame; - OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); - while ((frame = grabber.grab()) != null) { - if (frame.image != null) { - Mat mat = converter.convert(frame); - Mat rotated = new Mat(); - switch (angle) { - case 90: - opencv_core.transpose(mat, rotated); - opencv_core.flip(rotated, rotated, 1); - break; - case 180: - opencv_core.flip(mat, rotated, -1); - break; - case 270: - opencv_core.transpose(mat, rotated); - opencv_core.flip(rotated, rotated, 0); - break; - default: - rotated = mat.clone(); - break; - } - frame = converter.convert(rotated); - recorder.record(frame); - } - } - recorder.stop(); - recorder.release(); - grabber.stop(); - grabber.release(); - } - -} diff --git a/examples/face-example/pom.xml b/examples/face-example/pom.xml index 5562054..08db1dd 100644 --- a/examples/face-example/pom.xml +++ b/examples/face-example/pom.xml @@ -55,11 +55,7 @@ commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +69,6 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - com.alibaba @@ -101,7 +92,7 @@ ai.djl.pytorch pytorch-jni - 2.7.1-0.34.0 + 2.5.1-0.32.0 runtime @@ -138,7 +129,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.7.1 + 2.5.1 runtime @@ -176,9 +167,18 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.7.1 + 2.5.1 runtime + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + + + @@ -213,11 +213,13 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.7.1 + 2.5.1 runtime + + org.bytedeco @@ -247,22 +249,6 @@ ${javacv.platform.linux-arm64} - - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.5.1 - runtime - - - - ai.djl.pytorch - pytorch-native-cpu - linux-aarch64 - runtime - 2.7.1 - - @@ -294,16 +280,10 @@ - aliyunmaven - 阿里云公共仓库 - https://maven.aliyun.com/repository/public - - true - - - false - + central + https://repo1.maven.org/maven2/ + diff --git a/examples/face-example/src/main/java/smartai/examples/face/PythonTranslator.java b/examples/face-example/src/main/java/smartai/examples/face/PythonTranslator.java deleted file mode 100644 index 542118c..0000000 --- a/examples/face-example/src/main/java/smartai/examples/face/PythonTranslator.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance - * with the License. A copy of the License is located at - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES - * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ -package smartai.examples.face; - -import ai.djl.ModelException; -import ai.djl.inference.Predictor; -import ai.djl.modality.Classifications; -import ai.djl.modality.Input; -import ai.djl.modality.Output; -import ai.djl.modality.cv.Image; -import ai.djl.ndarray.NDList; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.translate.NoBatchifyTranslator; -import ai.djl.translate.TranslateException; -import ai.djl.translate.TranslatorContext; -import ai.djl.util.JsonUtils; -import ai.djl.util.Utils; - -import com.google.gson.reflect.TypeToken; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Type; -import java.net.URL; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class PythonTranslator implements NoBatchifyTranslator { - - private ZooModel model; - private Predictor predictor; - - @Override - public void prepare(TranslatorContext ctx) throws ModelException, IOException { - if (predictor == null) { - Criteria criteria = - Criteria.builder() - .setTypes(Input.class, Output.class) - .optModelPath(Paths.get("src/test/python")) - .optEngine("Python") - .build(); - model = criteria.loadModel(); - predictor = model.newPredictor(); - } - } - -// @Override -// public NDList processInput(TranslatorContext ctx, String url) -// throws IOException, TranslateException { -// Input input = new Input(); -// try (InputStream is = new URL(url).openStream()) { -// input.add("data", Utils.toByteArray(is)); -// } -// input.addProperty("Content-Type", "image/jpeg"); -// // calling preprocess() function in model.py -// input.addProperty("handler", "preprocess"); -// Output output = predictor.predict(input); -// if (output.getCode() != 200) { -// throw new TranslateException("Python preprocess() failed: " + output.getMessage()); -// } -// -// return output.getDataAsNDList(ctx.getNDManager()); -// } - - @Override - public NDList processInput(TranslatorContext ctx, byte[] image) - throws IOException, TranslateException { - Input input = new Input(); - input.add("data", image); - input.addProperty("Content-Type", "image/jpeg"); - // calling preprocess() function in model.py - input.addProperty("handler", "preprocess"); - Output output = predictor.predict(input); - if (output.getCode() != 200) { - throw new TranslateException("Python preprocess() failed: " + output.getMessage()); - } - return output.getDataAsNDList(ctx.getNDManager()); - } - - @Override - public Classifications processOutput(TranslatorContext ctx, NDList list) - throws TranslateException { - Input input = new Input(); - input.add("data", list); - // calling postprocess() function in processing.py - input.addProperty("handler", "postprocess"); - Output output = predictor.predict(input); - if (output.getCode() != 200) { - throw new TranslateException("Python postprocess() failed: " + output.getMessage()); - } - - String json = output.getData().getAsString(); - System.out.println("json:" + json); - return null; - } - - public void close() { - if (predictor != null) { - predictor.close(); - model.close(); - predictor = null; - model = null; - } - } -} diff --git a/examples/face-example/src/main/java/smartai/examples/face/Test.java b/examples/face-example/src/main/java/smartai/examples/face/Test.java deleted file mode 100644 index 67b3870..0000000 --- a/examples/face-example/src/main/java/smartai/examples/face/Test.java +++ /dev/null @@ -1,99 +0,0 @@ -package smartai.examples.face; - -import ai.djl.Application; -import ai.djl.Device; -import ai.djl.MalformedModelException; -import ai.djl.inference.Predictor; -import ai.djl.modality.Classifications; -import ai.djl.modality.audio.Audio; -import ai.djl.modality.audio.AudioFactory; -import ai.djl.modality.audio.translator.SpeechRecognitionTranslatorFactory; -import ai.djl.repository.Artifact; -import ai.djl.repository.MRL; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.repository.zoo.ModelZoo; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.translate.TranslateException; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; - -/** - * @author dwj - * @date 2025/7/29 - */ -@Slf4j -public class Test { - - public static void main(String[] args) throws ModelNotFoundException, MalformedModelException, IOException, TranslateException { -// PythonTranslator translator = new PythonTranslator(); -// Criteria criteria = -// Criteria.builder() -// .setTypes(byte[].class, Classifications.class) -// .optModelPath(Paths.get("/Users/wenjie/Documents/develop/model/arcfaceresnet100-11-int8.onnx")) -// .optEngine("OnnxRuntime") -// .optTranslator(translator) -// .build(); -// String path = "/Users/wenjie/Downloads/facetest/jsy.jpg"; -// try (ZooModel model = criteria.loadModel(); -// Predictor predictor = model.newPredictor()) { -// byte[] data = Files.readAllBytes(Paths.get(path)); -// Classifications ret = predictor.predict(data); -// System.out.println(ret); -// } -// -// // unload python model -// translator.close(); - - - // Load model. - // Wav2Vec2 model is a speech model that accepts a float array corresponding to the raw - // waveform of the speech signal. - -// String url = "/Users/wenjie/Downloads/20210601_u2++_conformer_exp/final.pt"; -// Criteria criteria = -// Criteria.builder() -// .setTypes(Audio.class, String.class) -//// .optModelUrls(url) -// .optModelPath(Paths.get(url)) -// .optDevice(Device.cpu()) // torchscript model only support CPU -// .optTranslatorFactory(new SpeechRecognitionTranslatorFactory()) -//// .optModelName("data.pkl") -// .optEngine("PyTorch") -// .build(); -// -// // Read in audio file -// String wave = "https://resources.djl.ai/audios/speech.wav"; -// Audio audio = AudioFactory.newInstance().fromUrl(wave); -// try (ZooModel model = criteria.loadModel(); -// Predictor predictor = model.newPredictor()) { -// String result = predictor.predict(audio); -// log.info("Result: {}", result); -// } - - boolean withArtifacts = - args.length > 0 && ("--artifact".equals(args[0]) || "-a".equals(args[0])); - if (!withArtifacts) { - log.info("============================================================"); - log.info("user ./gradlew listModel --args='-a' to show artifact detail"); - log.info("============================================================"); - } -// Map> models = ModelZoo.listModels(); -// for (Map.Entry> entry : models.entrySet()) { -// String appName = entry.getKey().toString(); -// for (Artifact artifact : entry.getValue()) { -// if (withArtifacts) { -// log.info("{} djl://{}", appName, artifact); -// } else { -// log.info("{} {}", appName, artifact); -// } -// } -// } - } -} diff --git a/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java b/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java deleted file mode 100644 index 64c94ff..0000000 --- a/examples/face-example/src/main/java/smartai/examples/face/VideoDemo.java +++ /dev/null @@ -1,25 +0,0 @@ -package smartai.examples.face; - -import cn.smartjavaai.common.utils.VideoUtils; -import org.bytedeco.ffmpeg.global.avcodec; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.FFmpegFrameRecorder; - -/** - * 视频预处理 - * @author dwj - * @date 2025/7/17 - */ -public class VideoDemo { - - public static void main(String[] args) { - try { - VideoUtils.rotateVideo("/Users/wenjie/Downloads/girl.mp4", "/Users/wenjie/Downloads/girl_rotate.mp4", 180,"mp4", avcodec.AV_CODEC_ID_H264); - } catch (FFmpegFrameRecorder.Exception e) { - throw new RuntimeException(e); - } catch (FFmpegFrameGrabber.Exception e) { - throw new RuntimeException(e); - } - } - -} diff --git a/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java index cfc2c18..d240f90 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java @@ -30,6 +30,7 @@ import java.util.List; /** * 人脸属性检测demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -91,23 +92,6 @@ public class FaceAttributeDetDemo { } } - /** - * 图片多人脸属性检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testFaceAttributeDetect3(){ - try { - FaceAttributeModel faceAttributeModel = getFaceAttributeModel(); - FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/iu_1.jpg"); - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); - } catch (Exception e) { - e.printStackTrace(); - } - //人脸检测 - - - - } /** * 图片单人脸人脸属性检测（基于已检测出的人脸区域和关键点） diff --git a/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java index b37e62e..b112dba 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java @@ -48,6 +48,8 @@ import java.util.List; /** * 表情识别demo * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j diff --git a/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java index 99f7192..8fe5a89 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java @@ -39,9 +39,8 @@ import java.nio.file.Paths; /** * 人脸检测模型demo - * 支持系统：windows 64位，linux 64位, macos M系列 - * 支持功能：人脸检测 * 模型下载地址：https://pan.baidu.com/s/1d2YlJ2YOdGn3Y-AegyAhmQ?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j diff --git a/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java index cc9d3d2..8cf5c5d 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java @@ -32,12 +32,10 @@ import java.io.IOException; import java.util.List; /** - * FaceNet人脸算法模型demo - * 支持系统：windows 64位，linux 64位，macOS M系列芯片 - * 支持功能：人脸特征提取、人脸比对（1：1）、人脸比对（1：N）、人脸注册 + * 人脸识别模型demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/4/11 */ @Slf4j public class FaceRecDemo { @@ -142,8 +140,8 @@ public class FaceRecDemo { */ public FaceRecModel getHighSpeedFaceRecModel(){ FaceRecConfig config = new FaceRecConfig(); - //高精度模型，速度慢 - config.setModelEnum(FaceRecModelEnum.SEETA_FACE6_MODEL); + //模型枚举 + config.setModelEnum(FaceRecModelEnum.SEETA_FACE6_LIGHT_MODEL); //模型路径，请下载模型并替换为本地路径：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 config.setModelPath("/Users/xxx/Documents/develop/model/sf3.0_models"); //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false diff --git a/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java index b1dedd8..8950c9d 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java @@ -54,8 +54,8 @@ import java.util.List; /** * 静态活体检测demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/1 */ @Slf4j public class LivenessDetDemo { diff --git a/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java index 77b53a6..10b6ed0 100644 --- a/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java @@ -36,6 +36,7 @@ import java.nio.file.Paths; /** * 人脸质量评估 demo * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j diff --git a/examples/objectdetection-example/.gitignore b/examples/objectdetection-example/.gitignore deleted file mode 100644 index 93dbf83..0000000 --- a/examples/objectdetection-example/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.idea -.idea/ -target -log -*.iml -/.settings/ -/logging.file_IS_UNDEFINED/ diff --git a/examples/objectdetection-example/output/object_detection_detected.png b/examples/objectdetection-example/output/object_detection_detected.png deleted file mode 100644 index bb2edcc..0000000 Binary files a/examples/objectdetection-example/output/object_detection_detected.png and /dev/null differ diff --git a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java deleted file mode 100644 index 330559c..0000000 --- a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java +++ /dev/null @@ -1,343 +0,0 @@ -package smartai.examples.objectdetection; - -import ai.djl.Application; -import ai.djl.MalformedModelException; -import ai.djl.ModelException; -import ai.djl.inference.Predictor; -import ai.djl.modality.Classifications; -import ai.djl.modality.cv.Image; -import ai.djl.modality.cv.ImageFactory; -import ai.djl.modality.cv.output.*; -import ai.djl.modality.cv.output.Rectangle; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.repository.zoo.ModelZoo; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; -import ai.djl.translate.TranslateException; -import cn.smartjavaai.common.config.Config; -import cn.smartjavaai.common.entity.DetectionInfo; -import cn.smartjavaai.common.entity.DetectionRectangle; -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.R; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.common.utils.OpenCVUtils; -import cn.smartjavaai.objectdetection.config.DetectorModelConfig; -import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; -import cn.smartjavaai.objectdetection.exception.DetectionException; -import cn.smartjavaai.objectdetection.model.DetectorModel; -import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import nu.pattern.OpenCV; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.opencv.core.Mat; -import org.opencv.core.Size; -import org.opencv.imgproc.Imgproc; -import org.opencv.videoio.VideoCapture; -import org.opencv.videoio.Videoio; - -import javax.imageio.ImageIO; -import javax.swing.*; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Paths; -import java.util.*; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - -/** - * 目标检测模型demo - * 支持功能：目标检测 - * 模型下载地址：https://pan.baidu.com/s/10aTOLBlR6EG-sq6g0OkAWg?pwd=1234 提取码: 1234 - * @author dwj - */ -@Slf4j -public class ObjectDetection { - - - //设备类型 - public static DeviceEnum device = DeviceEnum.CPU; - - public static void main(String[] args) throws ModelException, TranslateException, IOException { - Classifications classification = predict(); - log.info("{}", classification); - } - - - public static Classifications predict() throws IOException, ModelException, TranslateException { - - Config.setCachePath("/Users/wenjie/smartjavaai_cache"); - URL url = new URL("https://resources.djl.ai/images/action_dance.jpg"); - // Use DJL PyTorch model zoo model - Criteria criteria = - Criteria.builder() - .setTypes(URL.class, Classifications.class) - .optModelUrls( - "djl://ai.djl.mxnet/action_recognition") - .optEngine("MXNet") - .optProgress(new ProgressBar()) - .build(); - - try (ZooModel inception = criteria.loadModel(); - Predictor action = inception.newPredictor()) { - return action.predict(url); - } - } - - @BeforeClass - public static void beforeAll() throws IOException { - //修改缓存路径 -// Config.setCachePath("/Users/xxx/smartjavaai_cache"); - } - - - - /** - * 使用默认模型检测：YOLO11N - */ - @Test - public void objectDetection(){ - //默认cpu - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/object_detection.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 指定模型检测(19种模型可选) - */ - @Test - public void objectDetection2(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.SSD_300_RESNET50);//检测模型，目前支持19种预置模型 - config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL); - config.setModelPath("yolov11s"); - // 指定允许的类别 -// config.setAllowedClasses(Arrays.asList("person")); - //指定返回检测数量 - config.setTopK(100); - config.setDevice(device); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测并绘制检测结果 - */ - @Test - public void objectDetectionAndDraw(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - detectorModel.detectAndDraw("src/main/resources/object_detection.jpg","output/object_detection_detected.png"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 人脸检测并绘制检测结果,返回BufferedImage - */ - @Test - public void objectDetectionAndDraw2(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - String imagePath = "src/main/resources/object_detection.jpg"; - BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - BufferedImage detectedImage = detectorModel.detectAndDraw(image); - Assert.assertNotNull("detectedImage null", detectedImage); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - - /** - * 使用yolo官方模型检测物品识别 - */ - @Test - public void objectDetectionWithOfficialModel(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); -// config.setThreshold(0.3f); - //也支持YoloV8：YOLOV8_OFFICIAL 模型可以从文档中提供的地址下载 - config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL);//检测模型，目前支持19种模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("/Users/wenjie/Documents/develop/face_model/yolo11n.torchscript"); - config.setDevice(device); - config.putCustomParam("width", 640);//resize 宽 - config.putCustomParam("height", 640);// resize 高 - config.putCustomParam("resize", true); - config.putCustomParam("toTensor", true); - config.putCustomParam("applyRatio", true); - config.putCustomParam("threshold", 0.6f); - // for performance optimization maxBox parameter can reduce number of - // considered boxes from 8400 - config.putCustomParam("maxBox", 8400); -// config.putCustomParam("pad", 114d); -// List mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f); -// String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); -// config.putCustomParam("normalize", normalize); -// config.putCustomParam("flag", Image.Flag.COLOR); -// config.putCustomParam("pad", 114); - //一定要将yolo官方的类别文件：synset.txt（文档中下载）放在模型同目录下，否则报错 - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detect = detectorModel.detect("src/main/resources/object_detection.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 使用自己训练的模型检测 - */ - @Test - public void objectDetectionWithCustomModel(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - //也支持YoloV8：YOLOV8_CUSTOM 模型需要自己训练，训练教程可以查看文档 - config.setModelEnum(DetectorModelEnum.YOLOV12_CUSTOM);//自定义YOLOV12模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("/Users/xxx/Documents/develop/fire_model/best.onnx"); - config.putCustomParam("width", 640);//resize 宽 - config.putCustomParam("height", 640);// resize 高 - config.putCustomParam("nmsThreshold", 0.5f); - // 指定允许的类别 -// config.setAllowedClasses(Arrays.asList("person")); - //指定返回检测数量 - config.setTopK(100); - config.setDevice(device); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * tensorflow目标检测 - */ - @Test - public void objectDetection3(){ - try { - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.TENSORFLOW2_OFFICIAL); - config.setModelPath("/Users/wenjie/Documents/develop/model/tensorflow/ssd_mobilenet_v2_320x320_coco17_tpu-8"); -// config.putCustomParam("synsetUrl", "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt"); -// config.putCustomParam("synsetPath", "/Users/wenjie/Downloads/mscoco_label_map.pbtxt.txt"); - config.putCustomParam("synsetFileName", "mscoco.pbtxt"); - // 指定允许的类别 -// config.setAllowedClasses(Arrays.asList("person")); - //指定返回检测数量 - config.setTopK(100); - config.setDevice(device); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - detectorModel.detectAndDraw("src/main/resources/dog_bike_car.jpg", "output/dog_bike_car_detect.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - /** - * 摄像头目标检测 - * 注意事项：如果视频比较卡，可以使用轻量的检测模型 - */ - @Test - public void testDetectCamera(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - OpenCV.loadShared(); - VideoCapture capture = new VideoCapture(0); - if (!capture.isOpened()) { - System.out.println("No camera detected"); - return; - } - - double ratio = - capture.get(Videoio.CAP_PROP_FRAME_WIDTH) - / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); - Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); - int height = (int) (screenSize.height * 0.65f); - int width = (int) (height * ratio); - if (width > screenSize.width) { - width = screenSize.width; - } - - Mat image = new Mat(); - boolean captured = false; - for (int i = 0; i < 10; ++i) { - captured = capture.read(image); - if (captured) { - break; - } - - try { - Thread.sleep(50); - } catch (InterruptedException ignore) { - // ignore - } - } - if (!captured) { - JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); - } - ViewerFrame frame = new ViewerFrame(width, height); - ImageFactory factory = ImageFactory.getInstance(); - Size size = new Size(width, height); - - while (capture.isOpened()) { - if (!capture.read(image)) { - break; - } - Mat resizeImage = new Mat(); - Imgproc.resize(image, resizeImage, size); - Image img = factory.fromImage(resizeImage); - BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); - DetectionResponse detectedResult = detectorModel.detect(bufferedImage); - if (Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().size() == 0){ - log.debug("未检测到物体"); - continue; - } - for(DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){ - DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); - String text = detectionInfo.getObjectDetInfo().getClassName(); - ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.RED); - } - frame.showImage(bufferedImage); - } - - capture.release(); - System.exit(0); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - -} diff --git a/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF b/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF deleted file mode 100644 index c9a4764..0000000 --- a/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF +++ /dev/null @@ -1,3 +0,0 @@ -Manifest-Version: 1.0 -Main-Class: smartai.examples.objectdetection.ObjectDetection - diff --git a/examples/ocr-examples/output/plate_recognized2.jpg b/examples/ocr-examples/output/plate_recognized2.jpg index c69dec0..468a416 100644 Binary files a/examples/ocr-examples/output/plate_recognized2.jpg and b/examples/ocr-examples/output/plate_recognized2.jpg differ diff --git a/examples/ocr-examples/pom.xml b/examples/ocr-examples/pom.xml index 82dd180..0f31ad8 100644 --- a/examples/ocr-examples/pom.xml +++ b/examples/ocr-examples/pom.xml @@ -34,7 +34,7 @@ cn.smartjavaai - smartjavaai-bom + bom ${smartjavaai.version} pom @@ -50,16 +50,13 @@ commons-cli 1.9.0 + commons-io commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +70,6 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - com.alibaba @@ -96,14 +88,14 @@ cn.smartjavaai - smartjavaai-ocr + ocr ai.djl.pytorch pytorch-jni - 2.7.1-0.34.0 + 2.5.1-0.32.0 runtime @@ -140,7 +132,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.7.1 + 2.5.1 runtime @@ -178,7 +170,14 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.7.1 + 2.5.1 + runtime + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 runtime @@ -215,7 +214,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.7.1 + 2.5.1 runtime @@ -249,13 +248,7 @@ ${javacv.platform.linux-arm64} - - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.7.1 - runtime - + diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java index e089bba..0d05cb2 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDetectionDemo.java @@ -29,6 +29,7 @@ import java.util.List; /** * OCR 文本检测 示例 * 模型下载地址：https://pan.baidu.com/s/15Noz2xHQzqMQSl1B19BobQ?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -52,7 +53,7 @@ public class OcrDetectionDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java index d3f74b0..6175653 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrDirectionDetDemo.java @@ -22,8 +22,8 @@ import java.util.List; /** * OCR 行文本方向检测 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class OcrDirectionDetDemo { @@ -43,7 +43,7 @@ public class OcrDirectionDetDemo { */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） directionModelConfig.setModelPath("/Users/xxx/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); @@ -58,7 +58,7 @@ public class OcrDirectionDetDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java index 1f2ca9f..e938ac4 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/common/OcrRecognizeDemo.java @@ -36,8 +36,8 @@ import java.util.List; /** * OCR 文本识别 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class OcrRecognizeDemo { @@ -58,7 +58,7 @@ public class OcrRecognizeDemo { */ public OcrCommonRecModel getRecModel(){ OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） recModelConfig.setRecModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); @@ -73,7 +73,7 @@ public class OcrRecognizeDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); @@ -87,7 +87,7 @@ public class OcrRecognizeDemo { */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） directionModelConfig.setModelPath("/Users/xxx/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); @@ -102,7 +102,7 @@ public class OcrRecognizeDemo { */ public OcrCommonRecModel getRecModelWithDirection() { OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java index 9ee74b8..b9319b7 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/plate/PlateRecDemo.java @@ -17,11 +17,17 @@ import lombok.extern.slf4j.Slf4j; import org.junit.BeforeClass; import org.junit.Test; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.List; /** + * 车牌识别demo + * 模型下载地址：https://pan.baidu.com/s/1YEP56UqYcL-Op80M6JAreA?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j @@ -42,8 +48,10 @@ public class PlateRecDemo { */ public PlateDetModel getPlateDetModel() { PlateDetModelConfig config = new PlateDetModelConfig(); + //车牌检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(PlateDetModelEnum.YOLOV5); - config.setModelPath("/Users/xxx/Documents/develop/model/plate/yolov5_plate_detect.onnx"); + //下载模型并替换本地路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/plate/yolov5_plate_detect.onnx"); config.setDevice(device); return PlateModelFactory.getInstance().getDetModel(config); } @@ -54,15 +62,22 @@ public class PlateRecDemo { */ public PlateRecModel getPlateRecModel() { PlateRecModelConfig recModelConfig = new PlateRecModelConfig(); + //车牌识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setModelEnum(PlateRecModelEnum.PLATE_REC_CRNN); - recModelConfig.setModelPath("/Users/xxx/Documents/develop/model/plate/plate_rec_color.onnx"); + //下载模型并替换本地路径 + recModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/plate/plate_rec_color.onnx"); + //指定车牌检测模型 recModelConfig.setPlateDetModel(getPlateDetModel()); return PlateModelFactory.getInstance().getRecModel(recModelConfig); } + /** + * 车牌识别 + */ @Test public void testDetect() { PlateRecModel plateRecModel = getPlateRecModel(); + //识别车号 R> result = plateRecModel.recognize("src/main/resources/plate/Quicker_20220930_180856.png"); if(result.isSuccess()){ log.info("车牌识别结果：{}", JsonUtils.toJson(result.getData())); @@ -71,9 +86,13 @@ public class PlateRecDemo { } } + /** + * 车牌识别及绘制结果 + */ @Test public void recognizeAndDraw() { PlateRecModel plateRecModel = getPlateRecModel(); + //识别车号并绘制结果 R result = plateRecModel.recognizeAndDraw("src/main/resources/plate/single_green.jpg", "output/plate_recognized2.jpg"); if(result.isSuccess()){ log.info("车牌识别成功"); @@ -83,5 +102,28 @@ public class PlateRecDemo { } + /** + * 车牌识别及绘制结果 + */ + @Test + public void recognizeAndDraw2() { + try { + PlateRecModel plateRecModel = getPlateRecModel(); + BufferedImage image = null; + String imagePath = "src/main/resources/plate/Quicker_20220930_180856.png"; + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + //可以根据后续业务场景使用detectedImage + R detectedImage = plateRecModel.recognizeAndDraw(image); + if(detectedImage.isSuccess()){ + log.info("车牌识别成功"); + ImageUtils.saveImage(detectedImage.getData(), "output/plate_recognized2.jpg"); + }else{ + log.error("车牌识别失败：{}", detectedImage.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } diff --git a/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java index bdbbb2b..c50ad68 100644 --- a/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/table/TableRecDemo.java @@ -38,8 +38,8 @@ import java.util.List; /** * OCR 表格识别 示例 * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/5/25 */ @Slf4j public class TableRecDemo { @@ -60,7 +60,7 @@ public class TableRecDemo { */ public OcrCommonRecModel getRecModel(){ OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定文本识别模型 + //指定文本识别模型，切换模型需要同时修改modelEnum及modelPath recModelConfig.setRecModelEnum(CommonRecModelEnum.PP_OCR_V5_MOBILE_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） recModelConfig.setRecModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_rec_infer/PP-OCRv5_mobile_rec_infer.onnx"); @@ -75,7 +75,7 @@ public class TableRecDemo { */ public OcrCommonDetModel getDetectionModel() { OcrDetModelConfig config = new OcrDetModelConfig(); - //指定检测模型 + //指定检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(CommonDetModelEnum.PP_OCR_V5_MOBILE_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-OCRv5_mobile_det_infer/PP-OCRv5_mobile_det_infer.onnx"); @@ -90,7 +90,7 @@ public class TableRecDemo { */ public OcrDirectionModel getDirectionModel(){ DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath directionModelConfig.setModelEnum(DirectionModelEnum.PP_LCNET_X0_25); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） directionModelConfig.setModelPath("/Users/wenjie/Documents/develop/model/ocr/PP-LCNet_x0_25_textline_ori_infer/PP-LCNet_x0_25_textline_ori_infer.onnx"); @@ -104,7 +104,7 @@ public class TableRecDemo { */ public TableStructureModel getTableStructureModel(){ TableStructureConfig config = new TableStructureConfig(); - //指定行文本方向检测模型 + //指定行文本方向检测模型，切换模型需要同时修改modelEnum及modelPath config.setModelEnum(TableStructureModelEnum.SLANET_PLUS); //指定行文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） config.setModelPath("/Users/wenjie/Documents/develop/model/ocr/slanet-plus/slanet-plus.onnx"); diff --git a/examples/speech-examples/pom.xml b/examples/speech-examples/pom.xml index cf980e1..e2b1cd9 100644 --- a/examples/speech-examples/pom.xml +++ b/examples/speech-examples/pom.xml @@ -22,7 +22,7 @@ cn.smartjavaai - smartjavaai-bom + bom ${smartjavaai.version} pom @@ -43,11 +43,6 @@ commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - org.testng testng @@ -61,11 +56,7 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - + com.alibaba @@ -84,7 +75,7 @@ cn.smartjavaai - smartjavaai-speech + speech com.microsoft.onnxruntime diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java index cd6d4b6..7992a39 100644 --- a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java @@ -33,17 +33,17 @@ import java.io.InputStream; import java.nio.file.Paths; /** - * * 语音识别demo + * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ * @author dwj - * @date 2025/8/6 */ @Slf4j public class SpeechRecognizeDemo { /** * 获取Whisper模型 - * 模型下载网盘：通过网盘分享的文件：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 + * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 * 更多模型下载地址：https://huggingface.co/ggerganov/whisper.cpp/tree/main * @return */ @@ -192,12 +192,12 @@ public class SpeechRecognizeDemo { * 每个模型只支持一种语言，请下载对应语音的模型，模型下载地址：https://alphacephei.com/vosk/models * 将模型解压后，将模型目录位置填写到此处 */ - config.setModelPath("/Users/xxx/Documents/develop/model/speech/vosk-model-cn-0.22"); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/vosk-model-cn-0.22"); /** * macos m系列芯片需要手动下载依赖库，并指定位置，其他平台不需要 * 下载地址：https://pan.baidu.com/s/1LZ_EX1XdTTp_f5ruud82MA?pwd=1234 提取码: 1234 */ -// config.setLibPath(Paths.get("/Users/wenjie/Downloads/vosk-arrch64-dylib-main/libvosk.dylib")); + config.setLibPath(Paths.get("/Users/wenjie/Downloads/vosk-arrch64-dylib-main/libvosk.dylib")); return SpeechRecognizerFactory.getInstance().getModel(config); } diff --git a/examples/translation-example/pom.xml b/examples/translation-example/pom.xml index f90843d..1ae1317 100644 --- a/examples/translation-example/pom.xml +++ b/examples/translation-example/pom.xml @@ -34,7 +34,7 @@ cn.smartjavaai - smartjavaai-bom + bom ${smartjavaai.version} pom @@ -55,11 +55,7 @@ commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +69,7 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - + com.alibaba @@ -94,14 +86,14 @@ cn.smartjavaai - smartjavaai-translate + translate ai.djl.pytorch pytorch-jni - 2.7.1-0.34.0 + 2.5.1-0.32.0 runtime @@ -112,7 +104,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.7.1 + 2.5.1 runtime @@ -124,7 +116,15 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.7.1 + 2.5.1 + runtime + + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 runtime @@ -134,23 +134,23 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.7.1 + 2.5.1 runtime - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.7.1 + pytorch-native-cpu + linux-aarch64 runtime + 2.5.1 + @@ -179,16 +179,21 @@ + + + + + + + + + + + + - aliyunmaven - 阿里云公共仓库 - https://maven.aliyun.com/repository/public - - true - - - false - + central + https://repo1.maven.org/maven2/ diff --git a/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java index 8e17fad..b667123 100644 --- a/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java +++ b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java @@ -18,34 +18,73 @@ import org.junit.Test; import java.io.IOException; /** - * 翻译Demo - * 支持 Meta AI 开源的 NLLB-200 模型，实现 200 多种语言之间的高质量互译。 - * NLLB-200官网地址：https://github.com/facebookresearch/fairseq/tree/nllb + * 机器翻译Demo * 模型下载地址：https://pan.baidu.com/s/1wf7btnb4cyBFv7DB7baHnw?pwd=1234 提取码: 1234 + * 开发文档：http://doc.smartjavaai.cn/ * @author dwj */ @Slf4j public class TranslationDemo { + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + @BeforeClass public static void beforeAll() throws IOException { //修改缓存路径 // Config.setCachePath("/Users/xxx/smartjavaai_cache"); } + /** + * 获取模型(NLLB) + * @return + */ + public TranslationModel getNllbModel() { + TranslationModelConfig config = new TranslationModelConfig(); + //指定翻译模型：NLLB,切换模型需同时修改modelEnum及modelPath + config.setModelEnum(TranslationModeEnum.NLLB_MODEL); + //指定模型路径，需将模型路径修改为本地的模型路径 + config.setModelPath("/Users/xxx/Documents/develop/model/trans/traced_translation_cpu.pt"); + config.setDevice(DeviceEnum.CPU); + return TranslationModelFactory.getInstance().getModel(config); + } /** - * 翻译 + * 获取中文模型(OPUS 中文->英文) + * @return + */ + public TranslationModel getOPUSModelZH_EN() { + TranslationModelConfig config = new TranslationModelConfig(); + //指定翻译模型,切换模型需同时修改modelEnum及modelPath + config.setModelEnum(TranslationModeEnum.OPUS_MT_ZH_EN); + //指定模型路径，需将模型路径修改为本地的模型路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/trans/opus-mt-zh-en/traced_translation.pt"); + config.setDevice(DeviceEnum.CPU); + return TranslationModelFactory.getInstance().getModel(config); + } + + /** + * 获取中文模型(OPUS 英文->中文) + * @return + */ + public TranslationModel getOPUSModelEN_ZH() { + TranslationModelConfig config = new TranslationModelConfig(); + //指定翻译模型,切换模型需同时修改modelEnum及modelPath + config.setModelEnum(TranslationModeEnum.OPUS_MT_EN_ZH); + //指定模型路径，需将模型路径修改为本地的模型路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/trans/opus-mt-en-zh/traced_translation.pt"); + config.setDevice(DeviceEnum.CPU); + return TranslationModelFactory.getInstance().getModel(config); + } + + + /** + * 翻译(nllb模型) */ @Test - public void translate() { + public void nllbTranslate() { try { - TranslationModelConfig config = new TranslationModelConfig(); - //指定翻译模型：NLLB - config.setModelEnum(TranslationModeEnum.NLLB_MODEL); - //指定模型路径，需将模型路径修改为本地的模型路径 - config.setModelPath("/Users/xxx/Documents/develop/model/trans/traced_translation_cpu.pt"); - TranslationModel translationModel = TranslationModelFactory.getInstance().getModel(config); + TranslationModel translationModel = getNllbModel(); //翻译参数 TranslateParam translateParam = new TranslateParam(); //输入文字 @@ -74,28 +113,16 @@ public class TranslationDemo { } /** - * GPU 翻译 + * 翻译(opus 中文->英文) */ @Test - public void translateGpu() { + public void opusTranslate1() { try { - TranslationModelConfig config = new TranslationModelConfig(); - //指定翻译模型：NLLB - config.setModelEnum(TranslationModeEnum.NLLB_MODEL); - //指定设备：GPU - config.setDevice(DeviceEnum.GPU); - //指定模型路径，需将模型路径修改为本地的 GPU 模型路径 - config.setModelPath("/Users/xxx/Documents/develop/model/trans/traced_translation_gpu.pt"); - //获取翻译模型 - TranslationModel translationModel = TranslationModelFactory.getInstance().getModel(config); + TranslationModel translationModel = getOPUSModelZH_EN(); //翻译参数 TranslateParam translateParam = new TranslateParam(); //输入文字 - translateParam.setInput("你好，欢迎使用SmartJavaAI！"); - //源语言：中文 - translateParam.setSourceLanguage(LanguageCode.ZHO_HANS); - //目标语言：韩语 - translateParam.setTargetLanguage(LanguageCode.ENG_LATN); + translateParam.setInput("SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 JAVA AI算法工具包"); R result = translationModel.translate(translateParam); if(result.isSuccess()){ log.info("翻译结果：{}", result.getData()); @@ -107,4 +134,28 @@ public class TranslationDemo { } } + /** + * 翻译(opus 英文->中文) + */ + @Test + public void opusTranslate2() { + try { + TranslationModel translationModel = getOPUSModelEN_ZH(); + //翻译参数 + TranslateParam translateParam = new TranslateParam(); + //输入文字 + translateParam.setInput("You don't have to be machine learning expert to get started"); + R result = translationModel.translate(translateParam); + if(result.isSuccess()){ + log.info("翻译结果：{}", result.getData()); + }else{ + log.error("翻译失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + } diff --git a/examples/objectdetection-example/README.md b/examples/vision-example/README.md similarity index 100% rename from examples/objectdetection-example/README.md rename to examples/vision-example/README.md diff --git a/examples/objectdetection-example/pom.xml b/examples/vision-example/pom.xml similarity index 78% rename from examples/objectdetection-example/pom.xml rename to examples/vision-example/pom.xml index d5187f1..ebca5de 100644 --- a/examples/objectdetection-example/pom.xml +++ b/examples/vision-example/pom.xml @@ -14,7 +14,7 @@ UTF-8 1.0.24 - smartai.examples.objectdetection.ObjectDetection + smartai.examples.vision.ObjectDetectionDemo 1.5.10 @@ -55,11 +55,7 @@ commons-io 2.17.0 - - org.apache.logging.log4j - log4j-slf4j2-impl - 2.24.1 - + org.testng testng @@ -73,11 +69,6 @@ logback-classic 1.2.3 - - org.slf4j - slf4j-api - 1.7.30 - com.alibaba @@ -101,7 +92,7 @@ ai.djl.pytorch pytorch-jni - 2.7.1-0.34.0 + 2.5.1-0.32.0 runtime @@ -138,10 +129,26 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.7.1 + 2.5.1 runtime + + ai.djl.tensorflow + tensorflow-native-cpu + win-x86_64 + runtime + 2.16.1 + + + + ai.djl.mxnet + mxnet-native-mkl + win-x86_64 + runtime + 1.9.1 + + @@ -176,7 +183,30 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.7.1 + 2.5.1 + runtime + + + + ai.djl.tensorflow + tensorflow-native-cpu + linux-x86_64 + runtime + 2.16.1 + + + ai.djl.mxnet + mxnet-native-mkl + linux-x86_64 + runtime + 1.9.1 + + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-x86_64} + 2.5.1 runtime @@ -213,7 +243,30 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.7.1 + 2.5.1 + runtime + + + + ai.djl.tensorflow + tensorflow-native-cpu + osx-aarch64 + 2.16.1 + runtime + + + + ai.djl.tensorflow + tensorflow-native-cpu + osx-aarch64 + 2.16.1 + runtime + + + ai.djl.mxnet + mxnet-native-mkl + osx-x86_64 + 1.9.1 runtime @@ -247,14 +300,6 @@ ${javacv.platform.linux-arm64} - - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-aarch64} - 2.7.1 - runtime - - @@ -285,23 +330,23 @@ + + + + + + + + + + + + - aliyunmaven - 阿里云公共仓库 - https://maven.aliyun.com/repository/public - - true - - - false - + central + https://repo1.maven.org/maven2/ - - - - - diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/ActionRecognizeDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/ActionRecognizeDemo.java new file mode 100644 index 0000000..7566d34 --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ActionRecognizeDemo.java @@ -0,0 +1,92 @@ +package smartai.examples.vision; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.action.config.ActionRecModelConfig; +import cn.smartjavaai.action.enums.ActionRecModelEnum; +import cn.smartjavaai.action.model.ActionRecModel; +import cn.smartjavaai.action.model.ActionRecModelFactory; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * 动作识别Demo + * 模型下载地址：https://pan.baidu.com/s/17doY4pgZM9EbtSIaoCWWCA?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class ActionRecognizeDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取动作识别模型 + * 注意事项： + * 1、不同模型支持的动作类别不同，请查看文档：http://doc.smartjavaai.cn + */ + public ActionRecModel getModel(){ + ActionRecModelConfig config = new ActionRecModelConfig(); + //动作识别模型切换时，需要同时更新 modelEnum 和 modelPath。其中部分 modelEnum 对应多个模型文件，可通过指定 modelPath 来选择具体的模型。 + config.setModelEnum(ActionRecModelEnum.INCEPTIONV3_KINETICS400_ONNX); + //模型所在路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/action/gluoncv-inceptionv3_kinetics400-695477a5.onnx"); + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + //指定允许的类别 +// config.setAllowedClasses(Arrays.asList("dancing_ballet")); + return ActionRecModelFactory.getInstance().getModel(config); + } + + + + /** + * 动作识别 + * 注意事项： + * 1、不同模型支持的动作类别不同，请查看文档：http://doc.smartjavaai.cn + * 2、图片中应该只包含单一动作人物 + * 3、动作识别，只做图片分类，并不做人物定位 + */ + @Test + public void actionRecognition(){ + try { + ActionRecModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/action/dance.jpg")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + if(CollectionUtils.isNotEmpty(result.getData().getClassNames())){ + //分数最高分类 + log.info("动作识别结果：{}", result.getData().best().toString()); + //按分数排序前5个结果 +// log.info("动作识别结果：{}", result.getData().topK(5).toString()); + }else{ + log.info("未识别到动作"); + } + }else{ + log.info("动作识别失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/InstanceSegDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/InstanceSegDemo.java new file mode 100644 index 0000000..d1d5a63 --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/InstanceSegDemo.java @@ -0,0 +1,135 @@ +package smartai.examples.vision; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import cn.smartjavaai.action.model.ActionRecModel; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.instanceseg.config.InstanceSegModelConfig; +import cn.smartjavaai.instanceseg.enums.InstanceSegModelEnum; +import cn.smartjavaai.instanceseg.model.InstanceSegModel; +import cn.smartjavaai.instanceseg.model.InstanceSegModelFactory; +import cn.smartjavaai.objectdetection.model.person.PersonDetModel; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +/** + * 实例分割 Demo + * 模型下载地址：https://pan.baidu.com/s/12nRRY9JFNDwLeg63jfBerA?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class InstanceSegDemo { + + + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取实例分割模型 + * 注意事项： + * 1、更多模型请查看文档：http://doc.smartjavaai.cn + */ + public InstanceSegModel getModel(){ + InstanceSegModelConfig config = new InstanceSegModelConfig(); + //实例分割模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(InstanceSegModelEnum.SEG_YOLO11N_ONNX); + //模型所在路径，synset.txt也需要放在同目录下 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/instance/yolo11n-seg-onnx/yolo11n-seg.onnx"); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("person","car")); + //指定返回检测数量 + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + return InstanceSegModelFactory.getInstance().getModel(config); + } + + + + /** + * 实例分割 + */ + @Test + public void instanceSegmentation(){ + try { + InstanceSegModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/dog_bike_car.jpg")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + log.info("实例分割结果：{}", result.getData()); + }else{ + log.info("实例分割失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 实例分割并绘制检测结果 + */ + @Test + public void instanceSegmentationAndDraw(){ + try { + InstanceSegModel detectorModel = getModel(); + R result = detectorModel.detectAndDraw("src/main/resources/dog_bike_car.jpg","output/dog_bike_car_detected.png"); + if(result.isSuccess()){ + log.info("实例分割结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("实例分割失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 实例分割并绘制检测结果 + */ + @Test + public void instanceSegmentationAndDraw2(){ + try { + InstanceSegModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/dog_bike_car.jpg")); + //可以根据后续业务场景使用detectedImage + R result = detectorModel.detectAndDraw(image); + if(result.isSuccess()){ + log.info("实例分割结果：{}", JSONObject.toJSONString(result.getData())); + //保存图片 + ImageUtils.saveImage(result.getData().getDrawnImage(), "dog_bike_car_detected.png", "output"); + }else{ + log.info("实例分割失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/ObbDetDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/ObbDetDemo.java new file mode 100644 index 0000000..08a2703 --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ObbDetDemo.java @@ -0,0 +1,129 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.instanceseg.config.InstanceSegModelConfig; +import cn.smartjavaai.instanceseg.enums.InstanceSegModelEnum; +import cn.smartjavaai.instanceseg.model.InstanceSegModelFactory; +import cn.smartjavaai.obb.config.ObbDetModelConfig; +import cn.smartjavaai.obb.enums.ObbDetModelEnum; +import cn.smartjavaai.obb.model.ObbDetModel; +import cn.smartjavaai.obb.model.ObbDetModelFactory; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * obb旋转框检测demo + * 模型下载地址：https://pan.baidu.com/s/1-tC0u-aha3tnMQwy8FKy1Q?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class ObbDetDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取旋转框检测模型 + * 注意事项： + * 1、更多模型请查看文档：http://doc.smartjavaai.cn + * 2、模型可检测物体请查看：模型同目录文件synset.txt + */ + public ObbDetModel getModel(){ + ObbDetModelConfig config = new ObbDetModelConfig(); + //旋转框检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(ObbDetModelEnum.YOLOV11); + //模型所在路径，synset.txt也需要放在同目录下 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/obb/yolo11n-obb.onnx"); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("plane","ship")); + //指定返回检测数量 + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + return ObbDetModelFactory.getInstance().getModel(config); + } + + + + /** + * 旋转框检测 + */ + @Test + public void obbDet(){ + try { + ObbDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/obb/boats.jpg")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + log.info("旋转框检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("旋转框检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 旋转框检测并绘制检测结果 + */ + @Test + public void obbDetAndDraw(){ + try { + ObbDetModel detectorModel = getModel(); + R result = detectorModel.detectAndDraw("src/main/resources/obb/boats.jpg","output/boats_detected.png"); + if(result.isSuccess()){ + log.info("旋转框检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("旋转框检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 旋转框检测并绘制检测结果 + */ + @Test + public void obbDetAndDraw2(){ + try { + ObbDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/obb/boats.jpg")); + //可以根据后续业务场景使用detectedImage + R result = detectorModel.detectAndDraw(image); + if(result.isSuccess()){ + log.info("旋转框检测结果：{}", JSONObject.toJSONString(result.getData())); + //保存图片 + ImageUtils.saveImage(result.getData().getDrawnImage(), "boats_obb_detected.png", "output"); + }else{ + log.info("旋转框检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/ObjectDetectionDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/ObjectDetectionDemo.java new file mode 100644 index 0000000..e70c61f --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ObjectDetectionDemo.java @@ -0,0 +1,420 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.util.JsonUtils; +import cn.hutool.core.date.LocalDateTimeUtil; +import cn.hutool.core.lang.UUID; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.VideoSourceType; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.objectdetection.config.DetectorModelConfig; +import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; +import cn.smartjavaai.objectdetection.model.DetectorModel; +import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; +import cn.smartjavaai.objectdetection.stream.StreamDetectionListener; +import cn.smartjavaai.objectdetection.stream.StreamDetector; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import org.opencv.videoio.Videoio; + +import javax.imageio.ImageIO; +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.*; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +/** + * 目标检测模型demo + * 模型下载地址：https://pan.baidu.com/s/10aTOLBlR6EG-sq6g0OkAWg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class ObjectDetectionDemo { + + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取目标检测模型 + * 注意事项： + * 1、更多模型请查看文档：http://doc.smartjavaai.cn/objectdetect.html + */ + public DetectorModel getModel(){ + DetectorModelConfig config = new DetectorModelConfig(); + //目标检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL_ONNX); + //模型所在路径，synset.txt也需要放在同目录下 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/object/yolov12/yolov12n.onnx"); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("person","car")); + //指定返回检测数量 + config.setTopK(100); + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + return ObjectDetectionModelFactory.getInstance().getModel(config); + } + + + + /** + * 目标检测 + */ + @Test + public void objectDetection(){ + try { + DetectorModel detectorModel = getModel(); + DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/object_detection.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 目标检测并绘制检测结果 + */ + @Test + public void objectDetectionAndDraw(){ + try { + DetectorModel detectorModel = getModel(); + detectorModel.detectAndDraw("src/main/resources/object_detection.jpg","output/object_detection_detected.png"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 目标检测并绘制检测结果,返回BufferedImage + */ + @Test + public void objectDetectionAndDraw2(){ + try { + DetectorModel detectorModel = getModel(); + String imagePath = "src/main/resources/object_detection.jpg"; + BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + //可以根据后续业务场景使用detectedImage + BufferedImage detectedImage = detectorModel.detectAndDraw(image); + Assert.assertNotNull("detectedImage null", detectedImage); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + /** + * 使用自己训练的模型检测 + */ + @Test + public void objectDetectionWithCustomModel(){ + try { + DetectorModelConfig config = new DetectorModelConfig(); + //目标检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(DetectorModelEnum.YOLOV12_CUSTOM_ONNX); + //模型所在路径，synset.txt也需要放在同目录下(分类文件，具体请看文档：http://doc.smartjavaai.cn/objectdetect.html#%E4%BD%BF%E7%94%A8%E8%87%AA%E5%B7%B1%E8%AE%AD%E7%BB%83%E7%9A%84%E6%A8%A1%E5%9E%8B%E6%A3%80%E6%B5%8B) + config.setModelPath("/Users/xxx/Documents/develop/fire_model/best.onnx"); + //模型训练时图片宽度 + config.putCustomParam("width", 640);//resize 宽 + //模型训练时图片高度 + config.putCustomParam("height", 640);// resize 高 + config.putCustomParam("nmsThreshold", 0.5f); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("person")); + //指定返回检测数量 + config.setTopK(100); + config.setDevice(device); + DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); + DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * tensorflow2目标检测 + * 注意事项： + * 1、百度网盘只提供部分模型，更多tensorflow模型可以前往官网下载：https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md + */ + @Test + public void objectDetection3(){ + try { + DetectorModelConfig config = new DetectorModelConfig(); + //指定模型枚举，可以通过modelPath指定不同tensorflow模型 + config.setModelEnum(DetectorModelEnum.TENSORFLOW2_OFFICIAL); + //模型路径，需解压模型压缩包，可以通过modelPath指定不同tensorflow模型 + config.setModelPath("/Users/wenjie/Documents/develop/model/tensorflow/ssd_mobilenet_v2_320x320_coco17_tpu-8"); +// config.putCustomParam("synsetUrl", "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt"); +// config.putCustomParam("synsetPath", "/Users/wenjie/Downloads/mscoco_label_map.pbtxt.txt"); + //分类文件，需下载放入模型路径下 + config.putCustomParam("synsetFileName", "mscoco.pbtxt"); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("person")); + //指定返回检测数量 + config.setTopK(100); + config.setDevice(device); + DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); + DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); + //检测并保存绘制结果 + detectorModel.detectAndDraw("src/main/resources/dog_bike_car.jpg", "output/dog_bike_car_detect.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 视频流目标检测 + */ + @Test + public void testStream(){ + StreamDetector detector = new StreamDetector.Builder() + //视频源类型：支持视频流、本地摄像头、视频文件 + .sourceType(VideoSourceType.STREAM) + //视频流地址，支持rtsp、rtmp、http等常见视频流 + .streamUrl("rtsp://username:password@ip:port/Streaming/Channels/101") + //每隔多少帧检测一次（需要根据模型检测速度决定） + .frameDetectionInterval(10) + //目标检测模型 + .detectorModel(getModel()) + //回调函数：检测到指定目标时触发（getModel中可指定模型检测的物体） + .listener(new StreamDetectionListener() { + + /** + * 建议把耗时操作放到新线程里执行 + * @param detectionInfoList 目标信息列表 + * @param image 检测到的图片 + */ + @Override + public void onObjectDetected(List detectionInfoList, Image image) { + log.info("时间：" + LocalDateTimeUtil.now().toString()); + log.info("检测结果：{}", JsonUtils.toJson(detectionInfoList)); + //绘制检测结果 + OpenCVUtils.drawRectAndText(image, detectionInfoList); + //保存图片 + ImageUtils.saveImage(image, "test"+ UUID.fastUUID().toString() +".png","/Users/wenjie/Downloads"); + if (image != null){ + ((Mat)image.getWrappedImage()).release(); + } + } + + @Override + public void onStreamEnded() { + log.info("视频流检测结束"); + } + + @Override + public void onStreamDisconnected() { + log.info("视频流断开连接"); + } + }).build(); + detector.startDetection(); + //阻塞主线程 + CountDownLatch latch = new CountDownLatch(1); + try { + latch.await(); // 一直阻塞，直到被 countDown + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + + /** + * 本地摄像头目标检测 + */ + @Test + public void testLocalCamera(){ + StreamDetector detector = new StreamDetector.Builder() + //视频源类型：支持视频流、本地摄像头、视频文件 + .sourceType(VideoSourceType.CAMERA) + //摄像头序号 + .cameraIndex(0) + //每隔多少帧检测一次（需要根据模型检测速度决定） + .frameDetectionInterval(5) + //目标检测模型 + .detectorModel(getModel()) + //回调函数：检测到指定目标时触发（getModel中可指定模型检测的物体） + .listener(new StreamDetectionListener() { + @Override + public void onObjectDetected(List detectionInfoList, Image image) { + log.info("时间：" + LocalDateTimeUtil.now().toString()); + log.info("检测结果：{}", JsonUtils.toJson(detectionInfoList)); + //绘制检测结果 + OpenCVUtils.drawRectAndText(image, detectionInfoList); + //保存图片 + ImageUtils.saveImage(image, "test"+ UUID.fastUUID().toString() +".png","/Users/wenjie/Downloads"); + } + + @Override + public void onStreamEnded() { + log.info("视频流检测结束"); + } + + @Override + public void onStreamDisconnected() { + log.info("视频流断开连接"); + } + }).build(); + detector.startDetection(); + //阻塞主线程 + CountDownLatch latch = new CountDownLatch(1); + try { + latch.await(); // 一直阻塞，直到被 countDown + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * 视频文件目标检测 + */ + @Test + public void testVideoFile(){ + StreamDetector detector = new StreamDetector.Builder() + //视频源类型：支持视频流、本地摄像头、视频文件 + .sourceType(VideoSourceType.FILE) + //摄像头序号 + .streamUrl("girl.mp4") + //每隔多少帧检测一次（需要根据模型检测速度决定） + .frameDetectionInterval(5) + //目标检测模型 + .detectorModel(getModel()) + //同物体重复检测时间间隔，单位s + .repeatGap(5) + //回调函数：检测到指定目标时触发（getModel中可指定模型检测的物体） + .listener(new StreamDetectionListener() { + @Override + public void onObjectDetected(List detectionInfoList, Image image) { + log.info("时间：" + LocalDateTimeUtil.now().toString()); + log.info("检测结果：{}", JsonUtils.toJson(detectionInfoList)); + //绘制检测结果 + OpenCVUtils.drawRectAndText(image, detectionInfoList); + //保存图片 + ImageUtils.saveImage(image, "test"+ UUID.fastUUID().toString() +".png","/Users/wenjie/Downloads"); + } + + @Override + public void onStreamEnded() { + log.info("视频流检测结束"); + } + + @Override + public void onStreamDisconnected() { + log.info("视频流断开连接"); + } + }).build(); + detector.startDetection(); + //阻塞主线程 + CountDownLatch latch = new CountDownLatch(1); + try { + latch.await(); // 一直阻塞，直到被 countDown + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + + /** + * 摄像头目标检测并实时预览 + * 注意事项：如果视频比较卡，可以使用更轻量的检测模型 + */ + @Test + public void testDetectCamera(){ + try { + DetectorModel detectorModel = getModel(); + OpenCV.loadShared(); + VideoCapture capture = new VideoCapture(0); + if (!capture.isOpened()) { + System.out.println("No camera detected"); + return; + } + + double ratio = + capture.get(Videoio.CAP_PROP_FRAME_WIDTH) + / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + int height = (int) (screenSize.height * 0.65f); + int width = (int) (height * ratio); + if (width > screenSize.width) { + width = screenSize.width; + } + + Mat image = new Mat(); + boolean captured = false; + for (int i = 0; i < 10; ++i) { + captured = capture.read(image); + if (captured) { + break; + } + + try { + Thread.sleep(50); + } catch (InterruptedException ignore) { + // ignore + } + } + if (!captured) { + JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); + } + ViewerFrame frame = new ViewerFrame(width, height); + ImageFactory factory = ImageFactory.getInstance(); + Size size = new Size(width, height); + + while (capture.isOpened()) { + if (!capture.read(image)) { + break; + } + Mat resizeImage = new Mat(); + Imgproc.resize(image, resizeImage, size); + Image img = factory.fromImage(resizeImage); + BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); + DetectionResponse detectedResult = detectorModel.detect(bufferedImage); + if (Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().size() == 0){ + log.debug("未检测到物体"); + continue; + } + for(DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){ + DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); + String text = detectionInfo.getObjectDetInfo().getClassName(); + ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.RED); + } + frame.showImage(bufferedImage); + } + + capture.release(); + System.exit(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/PersonDetectDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/PersonDetectDemo.java new file mode 100644 index 0000000..26eb4bf --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/PersonDetectDemo.java @@ -0,0 +1,121 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.objectdetection.config.PersonDetModelConfig; +import cn.smartjavaai.objectdetection.enums.PersonDetectorModelEnum; +import cn.smartjavaai.objectdetection.model.person.PersonDetModel; +import cn.smartjavaai.objectdetection.model.person.PersonDetModelFactory; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; + +/** + * 行人检测案例 + * 模型下载地址：https://pan.baidu.com/s/1EWfExw7pYjKEH5uR5wf3Rw?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class PersonDetectDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取行人检测模型 + */ + public PersonDetModel getModel(){ + PersonDetModelConfig config = new PersonDetModelConfig(); + //行人检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(PersonDetectorModelEnum.YOLOV8_PERSON); + //模型所在路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/person/yolov8n-person.onnx"); + //指定返回检测数量 + config.setTopK(100); + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + return PersonDetModelFactory.getInstance().getModel(config); + } + + + + /** + * 行人检测 + */ + @Test + public void objectDetection(){ + try { + PersonDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/person/person.png")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + log.info("行人检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("行人检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 行人检测并绘制检测结果 + */ + @Test + public void objectDetectionAndDraw(){ + try { + PersonDetModel detectorModel = getModel(); + //保存绘制后图片以及返回检测结果 + R result = detectorModel.detectAndDraw("src/main/resources/person/person.png","output/person_detected.png"); + if(result.isSuccess()){ + log.info("行人检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("行人检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 行人检测并绘制检测结果 + */ + @Test + public void objectDetectionAndDraw2(){ + try { + PersonDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/person/person.png")); + //可以根据后续业务场景使用detectedImage + R result = detectorModel.detectAndDraw(image); + if(result.isSuccess()){ + log.info("行人检测结果：{}", JSONObject.toJSONString(result.getData())); + //保存图片 + ImageUtils.saveImage(result.getData().getDrawnImage(), "person_result.png", "output"); + }else{ + log.info("行人检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/PoseDetDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/PoseDetDemo.java new file mode 100644 index 0000000..bebe389 --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/PoseDetDemo.java @@ -0,0 +1,118 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.Joints; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.pose.config.PoseModelConfig; +import cn.smartjavaai.pose.enums.PoseModelEnum; +import cn.smartjavaai.pose.model.PoseDetModelFactory; +import cn.smartjavaai.pose.model.PoseModel; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; + +/** + * 姿态估计demo + * 模型下载地址：https://pan.baidu.com/s/1pPYyl1V2CpcMYCO8CJQHGg?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class PoseDetDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取姿态估计模型 + * 注意事项： + * 1、更多模型请查看文档：http://doc.smartjavaai.cn + * 2、模型可检测物体请查看：模型同目录文件synset.txt + */ + public PoseModel getModel(){ + PoseModelConfig config = new PoseModelConfig(); + //姿态估计模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(PoseModelEnum.YOLOV8N_POSE_PT); + //模型所在路径，synset.txt也需要放在同目录下 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/pose/yolo11n-pose-onnx/yolo11n-pose.onnx"); + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.25f); + return PoseDetModelFactory.getInstance().getModel(config); + } + + + + /** + * 姿态估计 + */ + @Test + public void poseDet(){ + try { + PoseModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/pose/pose_soccer.png")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + log.info("姿态估计结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("姿态估计失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 姿态估计并绘制检测结果 + */ + @Test + public void poseDetAndDraw(){ + try { + PoseModel detectorModel = getModel(); + R result = detectorModel.detectAndDraw("src/main/resources/pose/pose_soccer.png","output/pose_detected.png"); + if(result.isSuccess()){ + log.info("姿态估计结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("姿态估计失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 姿态估计并绘制检测结果 + */ + @Test + public void poseDetAndDraw2(){ + try { + PoseModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/pose/pose_soccer.png")); + //可以根据后续业务场景使用detectedImage + Image drawImage = detectorModel.detectAndDraw(image); + //保存图片 + ImageUtils.saveImage(drawImage, "pose_detected.png", "output"); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + +} diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/SemSegDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/SemSegDemo.java new file mode 100644 index 0000000..5b28193 --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/SemSegDemo.java @@ -0,0 +1,123 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.CategoryMask; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.semseg.config.SemSegModelConfig; +import cn.smartjavaai.semseg.enums.SemSegModelEnum; +import cn.smartjavaai.semseg.model.SemSegModel; +import cn.smartjavaai.semseg.model.SemSegModelFactory; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * 语义分割 Demo 通过网盘分享的文件：语义分割（semantic_segmentation） + * 模型下载地址：https://pan.baidu.com/s/18gs9E5h_d9imPmNLHuDo9A?pwd=1234 提取码: 1234 + * 文档地址：http://doc.smartjavaai.cn/ + * @author dwj + */ +@Slf4j +public class SemSegDemo { + + + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取语义分割模型 + * 注意事项： + * 1、更多模型请查看文档：http://doc.smartjavaai.cn + */ + public SemSegModel getModel(){ + SemSegModelConfig config = new SemSegModelConfig(); + //语义分割模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(SemSegModelEnum.DEEPLABV3); + //模型所在路径，synset.txt也需要放在同目录下 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/semseg/deeplabv3/deeplabv3.pt"); + // 指定允许的类别 +// config.setAllowedClasses(Arrays.asList("person","car")); + //指定返回检测数量 + config.setDevice(device); + return SemSegModelFactory.getInstance().getModel(config); + } + + + + /** + * 语义分割 + */ + @Test + public void semSeg(){ + try { + SemSegModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/dog_bike_car.jpg")); + R result = detectorModel.detect(image); + if(result.isSuccess()){ + log.info("语义分割结果：{}", result.getData()); + }else{ + log.info("语义分割失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语义分割并绘制检测结果 + */ + @Test + public void semSegAndDraw(){ + try { + SemSegModel detectorModel = getModel(); + R result = detectorModel.detectAndDraw("src/main/resources/dog_bike_car.jpg","output/dog_bike_car_semseg.png"); + if(result.isSuccess()){ + log.info("语义分割结果：{}", result.getData()); + }else{ + log.info("语义分割失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语义分割并绘制检测结果 + */ + @Test + public void semSegAndDraw2(){ + try { + SemSegModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/dog_bike_car.jpg")); + //可以根据后续业务场景使用detectedImage + Image dretectedImage = detectorModel.detectAndDraw(image); + //保存 + ImageUtils.saveImage(dretectedImage, "dog_bike_car_detected.png", "output"); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + + +} diff --git a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java b/examples/vision-example/src/main/java/smartai/examples/vision/ViewerFrame.java similarity index 98% rename from examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java rename to examples/vision-example/src/main/java/smartai/examples/vision/ViewerFrame.java index eb83ce4..0e0dafe 100644 --- a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ViewerFrame.java @@ -10,7 +10,7 @@ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. */ -package smartai.examples.objectdetection; +package smartai.examples.vision; import javax.swing.*; import java.awt.*; diff --git a/examples/vision-example/src/main/resources/META-INF/MANIFEST.MF b/examples/vision-example/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..3d44c28 --- /dev/null +++ b/examples/vision-example/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: smartai.examples.vision.ObjectDetectionDemo + diff --git a/examples/vision-example/src/main/resources/action/calling.jpeg b/examples/vision-example/src/main/resources/action/calling.jpeg new file mode 100644 index 0000000..ea56ea3 Binary files /dev/null and b/examples/vision-example/src/main/resources/action/calling.jpeg differ diff --git a/examples/vision-example/src/main/resources/action/dance.jpg b/examples/vision-example/src/main/resources/action/dance.jpg new file mode 100644 index 0000000..f011574 Binary files /dev/null and b/examples/vision-example/src/main/resources/action/dance.jpg differ diff --git a/examples/objectdetection-example/src/main/resources/dog_bike_car.jpg b/examples/vision-example/src/main/resources/dog_bike_car.jpg similarity index 100% rename from examples/objectdetection-example/src/main/resources/dog_bike_car.jpg rename to examples/vision-example/src/main/resources/dog_bike_car.jpg diff --git a/examples/objectdetection-example/src/main/resources/largest_selfie.jpg b/examples/vision-example/src/main/resources/largest_selfie.jpg similarity index 100% rename from examples/objectdetection-example/src/main/resources/largest_selfie.jpg rename to examples/vision-example/src/main/resources/largest_selfie.jpg diff --git a/examples/objectdetection-example/src/main/resources/logback.xml b/examples/vision-example/src/main/resources/logback.xml similarity index 100% rename from examples/objectdetection-example/src/main/resources/logback.xml rename to examples/vision-example/src/main/resources/logback.xml diff --git a/examples/vision-example/src/main/resources/obb/boats.jpg b/examples/vision-example/src/main/resources/obb/boats.jpg new file mode 100644 index 0000000..a04dcf8 Binary files /dev/null and b/examples/vision-example/src/main/resources/obb/boats.jpg differ diff --git a/examples/objectdetection-example/src/main/resources/object_detection.jpg b/examples/vision-example/src/main/resources/object_detection.jpg similarity index 100% rename from examples/objectdetection-example/src/main/resources/object_detection.jpg rename to examples/vision-example/src/main/resources/object_detection.jpg diff --git a/examples/vision-example/src/main/resources/person/person.png b/examples/vision-example/src/main/resources/person/person.png new file mode 100644 index 0000000..b9c20ec Binary files /dev/null and b/examples/vision-example/src/main/resources/person/person.png differ diff --git a/examples/vision-example/src/main/resources/pose/pose_soccer.png b/examples/vision-example/src/main/resources/pose/pose_soccer.png new file mode 100644 index 0000000..81fd19e Binary files /dev/null and b/examples/vision-example/src/main/resources/pose/pose_soccer.png differ diff --git a/examples/vision-example/src/main/resources/segmentation.jpg b/examples/vision-example/src/main/resources/segmentation.jpg new file mode 100644 index 0000000..e99898b Binary files /dev/null and b/examples/vision-example/src/main/resources/segmentation.jpg differ diff --git a/examples/vision-example/src/main/resources/zero/000000039769.jpg b/examples/vision-example/src/main/resources/zero/000000039769.jpg new file mode 100644 index 0000000..e131e8e Binary files /dev/null and b/examples/vision-example/src/main/resources/zero/000000039769.jpg differ diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java index 376f999..3311d44 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java @@ -1,5 +1,6 @@ package cn.smartjavaai.face.model.facedect; +import ai.djl.Device; import ai.djl.MalformedModelException; import ai.djl.engine.Engine; import ai.djl.inference.Predictor; @@ -18,6 +19,7 @@ import ai.djl.translate.NoopTranslator; import cn.smartjavaai.common.entity.*; import cn.smartjavaai.common.entity.Point; import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.pool.PredictorFactory; import cn.smartjavaai.common.utils.*; import cn.smartjavaai.face.config.FaceDetConfig; @@ -78,12 +80,16 @@ public class MtcnnFaceDetModel implements FaceDetModel{ throw new FaceException("MTCNN 模型需要指定存放模型文件的目录路径"); } try { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId()); + } Path pnetPath = modelPath.resolve("pnet_script.pt"); Path rnetPath = modelPath.resolve("rnet_script.pt"); Path onetPath = modelPath.resolve("onet_script.pt"); - pNetModel = getModel(pnetPath); - rNetModel = getModel(rnetPath); - oNetModel = getModel(onetPath); + pNetModel = getModel(pnetPath, device); + rNetModel = getModel(rnetPath, device); + oNetModel = getModel(onetPath, device); this.pnetPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(pNetModel)); this.rnetPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(rNetModel)); @@ -110,7 +116,7 @@ public class MtcnnFaceDetModel implements FaceDetModel{ * @throws MalformedModelException * @throws IOException */ - public ZooModel getModel(Path modelPath) throws ModelNotFoundException, MalformedModelException, IOException { + public ZooModel getModel(Path modelPath, Device device) throws ModelNotFoundException, MalformedModelException, IOException { Criteria criteria = Criteria.builder() .setTypes(NDList.class, NDList.class) @@ -118,6 +124,7 @@ public class MtcnnFaceDetModel implements FaceDetModel{ .optEngine("PyTorch") .optModelPath(modelPath) .optProgress(new ProgressBar()) + .optDevice(device) .build(); return criteria.loadModel(); } diff --git a/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java b/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java index 0c56f36..2646f8c 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java @@ -31,9 +31,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.Frame; -import org.bytedeco.javacv.Java2DFrameUtils; import org.opencv.core.Mat; import javax.imageio.ImageIO; diff --git a/ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java b/ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java index 480a0ae..5f5fe17 100644 --- a/ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java +++ b/ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java @@ -37,8 +37,6 @@ public class Yolo5PlateDetectTranslator implements Translator arguments) { confThreshold = arguments.containsKey("confThreshold") @@ -62,7 +60,8 @@ public class Yolo5PlateDetectTranslator implements Translator8 8 UTF-8 - 0.34.0 + 0.32.0 @@ -46,10 +46,10 @@ ai.djl api - - ai.djl - basicdataset - + + + + ai.djl model-zoo @@ -118,11 +118,11 @@ 1.7.30 - - org.slf4j - slf4j-simple - 1.7.30 - + + + + + commons-cli @@ -205,8 +205,6 @@ 1.5.10 - - diff --git a/vision/pom.xml b/vision/pom.xml index 8c906ab..67e9b20 100644 --- a/vision/pom.xml +++ b/vision/pom.xml @@ -36,32 +36,9 @@ ${project.version} - - org.bytedeco - javacpp - 1.5.10 - macosx-arm64 - - - org.bytedeco - ffmpeg - 6.1.1-1.5.10 - macosx-arm64 - - - org.bytedeco - openblas - 0.3.26-1.5.10 - macosx-arm64 - - - org.bytedeco - opencv - 4.9.0-1.5.10 - macosx-arm64 - + diff --git a/vision/src/main/java/cn/smartjavaai/action/criteria/ActionRecCriteriaFactory.java b/vision/src/main/java/cn/smartjavaai/action/criteria/ActionRecCriteriaFactory.java index c8eec8b..af99395 100644 --- a/vision/src/main/java/cn/smartjavaai/action/criteria/ActionRecCriteriaFactory.java +++ b/vision/src/main/java/cn/smartjavaai/action/criteria/ActionRecCriteriaFactory.java @@ -3,8 +3,10 @@ package cn.smartjavaai.action.criteria; import ai.djl.Device; import ai.djl.modality.Classifications; import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Translator; import cn.smartjavaai.action.config.ActionRecModelConfig; import cn.smartjavaai.action.enums.ActionRecModelEnum; import cn.smartjavaai.action.model.CommonActionTranslator; @@ -23,45 +25,48 @@ import java.util.concurrent.ConcurrentHashMap; public class ActionRecCriteriaFactory { + /** + * 创建动作识别Criteria + * @param config + * @return + */ public static Criteria createCriteria(ActionRecModelConfig config) { Device device = null; if(!Objects.isNull(config.getDevice())){ device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId()); } - Criteria criteria = null; - ConcurrentHashMap params = new ConcurrentHashMap(); - params.putAll(config.getCustomParams()); - if(config.getModelEnum() == ActionRecModelEnum.VIT_BASE_PATCH16_224){ - criteria = - Criteria.builder() - .setTypes(Image.class, Classifications.class) - .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : - config.getModelEnum().getModelUri()) - .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) - .optEngine("PyTorch") - .optDevice(device) - .optProgress(new ProgressBar()) - .build(); - }else { + Translator translator = getTranslator(config); + if(StringUtils.isBlank(config.getModelEnum().getModelUrl())){ + //检查模型路径 if (StringUtils.isBlank(config.getModelPath())){ throw new ActionException("请指定模型路径"); } - int width = 224; - int height = 224; - if(config.getModelEnum() == ActionRecModelEnum.INCEPTIONV3_KINETICS400){ - width = 299; - height = 299; - } - criteria = - Criteria.builder() - .setTypes(Image.class, Classifications.class) - .optTranslator(new CommonActionTranslator(width, height)) - .optEngine("OnnxRuntime") - .optModelPath(Paths.get(config.getModelPath())) - .optDevice(device) - .optProgress(new ProgressBar()) - .build(); } + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, Classifications.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : config.getModelEnum().getModelUrl()) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optTranslator(translator) + .optDevice(device) + .optProgress(new ProgressBar()) + .optEngine(config.getModelEnum().getEngine()) + .build(); return criteria; } + + /** + * 获取动作识别Translator + * @param config + * @return + */ + public static Translator getTranslator(ActionRecModelConfig config) { + Translator translator = null; + if(config.getModelEnum() == ActionRecModelEnum.INCEPTIONV1_KINETICS400_ONNX + || config.getModelEnum() == ActionRecModelEnum.INCEPTIONV3_KINETICS400_ONNX + || config.getModelEnum() == ActionRecModelEnum.INCEPTIONV3_KINETICS400_ONNX){ + translator =new CommonActionTranslator(config.getModelEnum().getInputWidth(), config.getModelEnum().getInputHeight()); + } + return translator; + } } diff --git a/vision/src/main/java/cn/smartjavaai/action/enums/ActionRecModelEnum.java b/vision/src/main/java/cn/smartjavaai/action/enums/ActionRecModelEnum.java index dbb0605..708c207 100644 --- a/vision/src/main/java/cn/smartjavaai/action/enums/ActionRecModelEnum.java +++ b/vision/src/main/java/cn/smartjavaai/action/enums/ActionRecModelEnum.java @@ -6,21 +6,33 @@ package cn.smartjavaai.action.enums; */ public enum ActionRecModelEnum { - VIT_BASE_PATCH16_224("djl://ai.djl.pytorch/Human-Action-Recognition-VIT-Base-patch16-224"), + VIT_BASE_PATCH16_224_DJL("PyTorch",0,0,"djl://ai.djl.pytorch/Human-Action-Recognition-VIT-Base-patch16-224"), - INCEPTIONV3_KINETICS400(""), + INCEPTIONV3_KINETICS400_ONNX("OnnxRuntime",299,299,""), - INCEPTIONV1_KINETICS400(""), + INCEPTIONV1_KINETICS400_ONNX("OnnxRuntime",224,224,""), - RESNET18_V1B_KINETICS400(""), + RESNET_V1B_KINETICS400_ONNX("OnnxRuntime",224,224,""); - RESNET34_V1B_KINETICS400(""), + /** + * 模型输入尺寸：宽 + */ + private final int inputWidth; - RESNET50_V1B_KINETICS400(""), + /** + * 模型输入尺寸：高 + */ + private final int inputHeight; - RESNET101_V1B_KINETICS400(""), + /** + * 模型地址 + */ + private final String modelUrl; - RESNET152_V1B_KINETICS400(""); + /** + * 模型引擎 + */ + private final String engine; /** * 根据名称获取枚举 (忽略大小写和下划线变体) @@ -35,14 +47,27 @@ public enum ActionRecModelEnum { throw new IllegalArgumentException("未知模型名称: " + name); } - private final String modelUri; - ActionRecModelEnum(String modelUri) { - this.modelUri = modelUri; + ActionRecModelEnum(String engine, int inputWidth, int inputHeight, String modelUrl) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + this.modelUrl = modelUrl; + this.engine = engine; } - public String getModelUri() { - return modelUri; + public int getInputWidth() { + return inputWidth; } + public int getInputHeight() { + return inputHeight; + } + + public String getModelUrl() { + return modelUrl; + } + + public String getEngine() { + return engine; + } } diff --git a/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModel.java b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModel.java index b708ee0..87f2188 100644 --- a/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModel.java +++ b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModel.java @@ -24,15 +24,6 @@ public interface ActionRecModel extends AutoCloseable{ */ void loadModel(ActionRecModelConfig config); - /** - * 动作检测 - * @param base64Image - * @return - */ - default R detectBase64(String base64Image){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - /** * 动作检测 * @param image diff --git a/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java new file mode 100644 index 0000000..9ddd820 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java @@ -0,0 +1,111 @@ +package cn.smartjavaai.action.model; + +import cn.smartjavaai.action.config.ActionRecModelConfig; +import cn.smartjavaai.action.enums.ActionRecModelEnum; +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.objectdetection.model.person.CommonPersonDetModel; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 动作识别 模型工厂 + * @author dwj + */ +@Slf4j +public class ActionRecModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile ActionRecModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private ActionRecModelFactory() {} + + // 双重检查锁定的单例方法 + public static ActionRecModelFactory getInstance() { + if (instance == null) { + synchronized (ActionRecModelFactory.class) { + if (instance == null) { + instance = new ActionRecModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public ActionRecModel getModel(ActionRecModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private ActionRecModel createFaceDetModel(ActionRecModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + ActionRecModel model = null; + try { + model = (ActionRecModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(ActionRecModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(ActionRecModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(ActionRecModelEnum.INCEPTIONV1_KINETICS400_ONNX, CommonActionRecModel.class); + registerAlgorithm(ActionRecModelEnum.INCEPTIONV3_KINETICS400_ONNX, CommonActionRecModel.class); + registerAlgorithm(ActionRecModelEnum.RESNET_V1B_KINETICS400_ONNX, CommonActionRecModel.class); + registerAlgorithm(ActionRecModelEnum.VIT_BASE_PATCH16_224_DJL, CommonActionRecModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/action/model/CommonActionRecModel.java b/vision/src/main/java/cn/smartjavaai/action/model/CommonActionRecModel.java index cd784c5..34808fd 100644 --- a/vision/src/main/java/cn/smartjavaai/action/model/CommonActionRecModel.java +++ b/vision/src/main/java/cn/smartjavaai/action/model/CommonActionRecModel.java @@ -68,26 +68,12 @@ public class CommonActionRecModel implements ActionRecModel{ } } - @Override - public R detectBase64(String base64Image) { - if(StringUtils.isBlank(base64Image)){ - return R.fail(R.Status.INVALID_IMAGE); - } - try { - byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); - Image image = ImageFactory.getInstance().fromInputStream(new ByteArrayInputStream(imageData)); - return detect(image); - } catch (IOException e) { - throw new DetectionException("读取图片异常", e); - } - } @Override public R detect(Image image) { Classifications classifications = detectCore(image); // 过滤 - if(config.getThreshold() > 0 && CollectionUtils.isNotEmpty(config.getAllowedClasses()) - && Objects.nonNull(classifications) && !classifications.items().isEmpty()){ + if(Objects.nonNull(classifications) && !classifications.items().isEmpty()){ classifications = new ClassificationFilter(config.getAllowedClasses(), config.getThreshold()).filter(classifications); } return R.ok(classifications); diff --git a/vision/src/main/java/cn/smartjavaai/action/model/CommonActionTranslator.java b/vision/src/main/java/cn/smartjavaai/action/model/CommonActionTranslator.java index 796bec1..9a2fa14 100644 --- a/vision/src/main/java/cn/smartjavaai/action/model/CommonActionTranslator.java +++ b/vision/src/main/java/cn/smartjavaai/action/model/CommonActionTranslator.java @@ -116,7 +116,6 @@ public class CommonActionTranslator implements Translator criteria = null; - ConcurrentHashMap params = new ConcurrentHashMap(); - params.putAll(config.getCustomParams()); +// ConcurrentHashMap params = new ConcurrentHashMap(); +// params.putAll(config.getCustomParams()); // YoloV5Translator.Builder builder = new YoloV5Translator.Builder() // .optSynsetArtifactName("synset.txt").setPipeline() - criteria = - Criteria.builder() - .setTypes(Image.class, DetectedObjects.class) - .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : - config.getModelEnum().getModelUri()) - .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) - .optDevice(device) - .optEngine("PyTorch") - .optTranslatorFactory(new YoloSegmentationTranslatorFactory2()) - .optProgress(new ProgressBar()) - .build(); + + if(config.getModelEnum() == InstanceSegModelEnum.SEG_MASK_RCNN){ + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : + config.getModelEnum().getModelUri()) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optDevice(device) + .optEngine(config.getModelEnum().getEngine()) + .optArgument("normalize","true") + .optArgument("synsetFileName","classes.txt") + .optTranslatorFactory(new InstanceSegmentationTranslatorFactory()) + .optProgress(new ProgressBar()) + .build(); + }else{ + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : + config.getModelEnum().getModelUri()) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optDevice(device) + .optArgument("width", config.getModelEnum().getInputWidth()) + .optArgument("height", config.getModelEnum().getInputHeight()) + .optArgument("resize", "true") + .optArgument("threshold", config.getThreshold()) + .optEngine(config.getModelEnum().getEngine()) + .optTranslatorFactory(new YoloSegmentationTranslatorFactory2()) + .optProgress(new ProgressBar()) + .build(); + } return criteria; } } diff --git a/vision/src/main/java/cn/smartjavaai/instanceseg/enums/InstanceSegModelEnum.java b/vision/src/main/java/cn/smartjavaai/instanceseg/enums/InstanceSegModelEnum.java index d87ccb6..611a8c6 100644 --- a/vision/src/main/java/cn/smartjavaai/instanceseg/enums/InstanceSegModelEnum.java +++ b/vision/src/main/java/cn/smartjavaai/instanceseg/enums/InstanceSegModelEnum.java @@ -6,15 +6,15 @@ package cn.smartjavaai.instanceseg.enums; */ public enum InstanceSegModelEnum { - SEG_YOLO11N_PYTORCH("djl://ai.djl.pytorch/yolo11n-seg"), + SEG_YOLO11N_PYTORCH("PyTorch", 640, 640, "djl://ai.djl.pytorch/yolo11n-seg"), - SEG_YOLOV8N_PYTORCH("djl://ai.djl.pytorch/yolo11n-seg"), + SEG_YOLOV8N_PYTORCH("PyTorch", 640, 640, "djl://ai.djl.pytorch/yolov8n-seg"), - SEG_YOLO11N_ONNX("djl://ai.djl.onnxruntime/yolo11n-seg"), + SEG_YOLO11N_ONNX("OnnxRuntime", 640, 640, "djl://ai.djl.onnxruntime/yolo11n-seg"), - SEG_YOLOV8N_ONNX("djl://ai.djl.onnxruntime/yolov8n-seg"), + SEG_YOLOV8N_ONNX("OnnxRuntime", 640, 640, "djl://ai.djl.onnxruntime/yolov8n-seg"), - SEG_MASK_RCNN("djl://ai.djl.mxnet/mask_rcnn"); + SEG_MASK_RCNN("MXNet", 0,0, "djl://ai.djl.mxnet/mask_rcnn"); /** @@ -30,14 +30,43 @@ public enum InstanceSegModelEnum { throw new IllegalArgumentException("未知模型名称: " + name); } + /** + * 模型输入尺寸：宽 + */ + private final int inputWidth; + + /** + * 模型输入尺寸：高 + */ + private final int inputHeight; + private final String modelUri; - InstanceSegModelEnum(String modelUri) { + /** + * 模型引擎 + */ + private final String engine; + + InstanceSegModelEnum(String engine, int inputWidth, int inputHeight, String modelUri) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; this.modelUri = modelUri; + this.engine = engine; } public String getModelUri() { return modelUri; } + public String getEngine() { + return engine; + } + + public int getInputWidth() { + return inputWidth; + } + + public int getInputHeight() { + return inputHeight; + } } diff --git a/vision/src/main/java/cn/smartjavaai/instanceseg/model/CommonInstanceSegModel.java b/vision/src/main/java/cn/smartjavaai/instanceseg/model/CommonInstanceSegModel.java index 6e3324f..d5c3ec5 100644 --- a/vision/src/main/java/cn/smartjavaai/instanceseg/model/CommonInstanceSegModel.java +++ b/vision/src/main/java/cn/smartjavaai/instanceseg/model/CommonInstanceSegModel.java @@ -113,6 +113,9 @@ public class CommonInstanceSegModel implements InstanceSegModel { @Override public R detectAndDraw(Image image) { DetectedObjects detectedObjects = detectCore(image); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } image.drawBoundingBoxes(detectedObjects); DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image); detectionResponse.setDrawnImage(image); @@ -124,6 +127,9 @@ public class CommonInstanceSegModel implements InstanceSegModel { try { Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); DetectedObjects detectedObjects = detectCore(img); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } img.drawBoundingBoxes(detectedObjects); img.save(Files.newOutputStream(Paths.get(outputPath)), "png"); DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, img); diff --git a/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java b/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java new file mode 100644 index 0000000..e5dbc2a --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java @@ -0,0 +1,111 @@ +package cn.smartjavaai.instanceseg.model; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.instanceseg.config.InstanceSegModelConfig; +import cn.smartjavaai.instanceseg.enums.InstanceSegModelEnum; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 实例分割 模型工厂 + * @author dwj + */ +@Slf4j +public class InstanceSegModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile InstanceSegModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private InstanceSegModelFactory() {} + + // 双重检查锁定的单例方法 + public static InstanceSegModelFactory getInstance() { + if (instance == null) { + synchronized (InstanceSegModelFactory.class) { + if (instance == null) { + instance = new InstanceSegModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public InstanceSegModel getModel(InstanceSegModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private InstanceSegModel createFaceDetModel(InstanceSegModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + InstanceSegModel model = null; + try { + model = (InstanceSegModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(InstanceSegModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(InstanceSegModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(InstanceSegModelEnum.SEG_YOLOV8N_ONNX, CommonInstanceSegModel.class); + registerAlgorithm(InstanceSegModelEnum.SEG_YOLOV8N_PYTORCH, CommonInstanceSegModel.class); + registerAlgorithm(InstanceSegModelEnum.SEG_YOLO11N_PYTORCH, CommonInstanceSegModel.class); + registerAlgorithm(InstanceSegModelEnum.SEG_YOLO11N_ONNX, CommonInstanceSegModel.class); + registerAlgorithm(InstanceSegModelEnum.SEG_MASK_RCNN, CommonInstanceSegModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/obb/config/ObbDetModelConfig.java b/vision/src/main/java/cn/smartjavaai/obb/config/ObbDetModelConfig.java new file mode 100644 index 0000000..2f24598 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/config/ObbDetModelConfig.java @@ -0,0 +1,59 @@ +package cn.smartjavaai.obb.config; + +import cn.smartjavaai.common.config.ModelConfig; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.obb.enums.ObbDetModelEnum; +import lombok.Data; + +import java.util.List; + +/** + * 旋转框模型参数配置 + * + * @author dwj + */ +@Data +public class ObbDetModelConfig extends ModelConfig { + + /** + * 模型 + */ + private ObbDetModelEnum modelEnum; + + + /** + * 模型路径 + */ + private String modelPath; + + + /** + * 允许的分类列表 + */ + private List allowedClasses; + + /** + * 置信度阈值 + */ + private float threshold = 0.25f; + + /** + * 按置信度分数排序后，最多保留的检测框数量 + */ + private int topK; + + + public ObbDetModelConfig() { + } + + public ObbDetModelConfig(ObbDetModelEnum modelEnum, DeviceEnum device) { + this.modelEnum = modelEnum; + setDevice(device); + } + + public ObbDetModelConfig(ObbDetModelEnum modelEnum) { + this.modelEnum = modelEnum; + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/criteria/ObbDetCriteriaFactory.java b/vision/src/main/java/cn/smartjavaai/obb/criteria/ObbDetCriteriaFactory.java new file mode 100644 index 0000000..9978b59 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/criteria/ObbDetCriteriaFactory.java @@ -0,0 +1,70 @@ +package cn.smartjavaai.obb.criteria; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Translator; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.obb.config.ObbDetModelConfig; +import cn.smartjavaai.obb.entity.ObbResult; +import cn.smartjavaai.obb.enums.ObbDetModelEnum; +import cn.smartjavaai.obb.exception.ObbDetException; +import cn.smartjavaai.obb.translator.YoloV11OddTranslator; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 旋转框Criteria工厂 + * @author dwj + */ +public class ObbDetCriteriaFactory { + + + /** + * 创建旋转框检测Criteria + * @param config + * @return + */ + public static Criteria createCriteria(ObbDetModelConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId()); + } + Translator translator = getTranslator(config); + //检查模型路径 + if (StringUtils.isBlank(config.getModelPath())){ + throw new ObbDetException("请指定模型路径"); + } + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, ObbResult.class) + .optModelPath(Paths.get(config.getModelPath())) + .optTranslator(translator) + .optDevice(device) + .optProgress(new ProgressBar()) + .optEngine(config.getModelEnum().getEngine()) + .build(); + return criteria; + } + + + /** + * 获取旋转框检测Translator + * @param config + * @return + */ + public static Translator getTranslator(ObbDetModelConfig config) { + Translator translator = null; + if (config.getModelEnum() == ObbDetModelEnum.YOLOV11){ + translator = YoloV11OddTranslator.builder() + .setImageSize(config.getModelEnum().getInputWidth(), config.getModelEnum().getInputHeight()) + .optThreshold(config.getThreshold() > 0 ? config.getThreshold() : 0.25f) + .optNmsThreshold(0.45f) + .optSynsetArtifactName("synset.txt").build(); + } + return translator; + } +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/entity/ObbResult.java b/vision/src/main/java/cn/smartjavaai/obb/entity/ObbResult.java new file mode 100644 index 0000000..f110c29 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/entity/ObbResult.java @@ -0,0 +1,20 @@ +package cn.smartjavaai.obb.entity; + +import lombok.Data; + +import java.util.List; + +/** + * odd检测结果 + * @author dwj + */ +@Data +public class ObbResult { + + private List rotatedBoxeList; + + + public ObbResult(List rotatedBoxeList) { + this.rotatedBoxeList = rotatedBoxeList; + } +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/entity/YoloRotatedBox.java b/vision/src/main/java/cn/smartjavaai/obb/entity/YoloRotatedBox.java new file mode 100644 index 0000000..ec9ac82 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/entity/YoloRotatedBox.java @@ -0,0 +1,98 @@ +package cn.smartjavaai.obb.entity; + +import cn.smartjavaai.common.entity.Point; + +import java.util.ArrayList; +import java.util.List; + +/** + * 旋转框 + * @author dwj + */ +public class YoloRotatedBox { + public float cx, cy, w, h, angle; + public float score; + + public String className; + + public YoloRotatedBox(float cx, float cy, float w, float h, float angle, String className, float score) { + this.cx = cx; + this.cy = cy; + this.w = w; + this.h = h; + this.angle = angle; + this.className = className; + this.score = score; + } + + public static double probiou(YoloRotatedBox b1, YoloRotatedBox b2, double eps) { + double[] c1 = covarianceMatrix(b1.w, b1.h, b1.angle); + double[] c2 = covarianceMatrix(b2.w, b2.h, b2.angle); + + double a1 = c1[0], b1v = c1[1], c1v = c1[2]; + double a2 = c2[0], b2v = c2[1], c2v = c2[2]; + + double x1 = b1.cx, y1 = b1.cy; + double x2 = b2.cx, y2 = b2.cy; + + double t1 = ((a1 + a2) * Math.pow(y1 - y2, 2) + (b1v + b2v) * Math.pow(x1 - x2, 2)) + / ((a1 + a2) * (b1v + b2v) - Math.pow(c1v + c2v, 2) + eps); + double t2 = ((c1v + c2v) * (x2 - x1) * (y1 - y2)) + / ((a1 + a2) * (b1v + b2v) - Math.pow(c1v + c2v, 2) + eps); + double t3 = Math.log(((a1 + a2) * (b1v + b2v) - Math.pow(c1v + c2v, 2)) + / (4 * Math.sqrt(a1 * b1v - Math.pow(c1v, 2)) * Math.sqrt(a2 * b2v - Math.pow(c2v, 2)) + eps) + eps); + + // 2. probiou 内部 clamp 严格对应 Python + double bd = 0.25 * t1 + 0.5 * t2 + 0.5 * t3; + bd = Math.max(Math.min(bd, 100.0), eps); + double hd = Math.sqrt(1.0 - Math.exp(-bd) + eps); + return 1 - hd; + } + + private static double[] covarianceMatrix(double w, double h, double r) { + double a = Math.pow(w, 2) / 12.0; + double b = Math.pow(h, 2) / 12.0; + double cos = Math.cos(r); + double sin = Math.sin(r); + + double aVal = a * cos * cos + b * sin * sin; + double bVal = a * sin * sin + b * cos * cos; + double cVal = (a - b) * sin * cos; + return new double[]{aVal, bVal, cVal}; + } + + + /** + * 转换为4点坐标 + * @return + */ + public List toPoints() { + double cos = Math.cos(angle); + double sin = Math.sin(angle); + + // vec1 = [w/2*cos, w/2*sin] + double vec1x = w / 2.0 * cos; + double vec1y = w / 2.0 * sin; + + // vec2 = [-h/2*sin, h/2*cos] + double vec2x = -h / 2.0 * sin; + double vec2y = h / 2.0 * cos; + + List points = new ArrayList<>(4); + + // pt1 = ctr + vec1 + vec2 + points.add(new Point((int) Math.round(cx + vec1x + vec2x), (int) Math.round(cy + vec1y + vec2y))); + + // pt2 = ctr + vec1 - vec2 + points.add(new Point((int) Math.round(cx + vec1x - vec2x), (int) Math.round(cy + vec1y - vec2y))); + + // pt3 = ctr - vec1 - vec2 + points.add(new Point((int) Math.round(cx - vec1x - vec2x), (int) Math.round(cy - vec1y - vec2y))); + + // pt4 = ctr - vec1 + vec2 + points.add(new Point((int) Math.round(cx - vec1x + vec2x), (int) Math.round(cy - vec1y + vec2y))); + + return points; + } + +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/enums/ObbDetModelEnum.java b/vision/src/main/java/cn/smartjavaai/obb/enums/ObbDetModelEnum.java new file mode 100644 index 0000000..ed85014 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/enums/ObbDetModelEnum.java @@ -0,0 +1,59 @@ +package cn.smartjavaai.obb.enums; + +/** + * 旋转框检测模型枚举 + * @author dwj + */ +public enum ObbDetModelEnum { + + YOLOV11("OnnxRuntime", 1024, 1024); + + /** + * 模型引擎 + */ + private final String engine; + + /** + * 模型输入尺寸：宽 + */ + private final int inputWidth; + + /** + * 模型输入尺寸：高 + */ + private final int inputHeight; + + + ObbDetModelEnum(String engine, int inputWidth, int inputHeight) { + this.engine = engine; + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + } + + public String getEngine() { + return engine; + } + + public int getInputWidth() { + return inputWidth; + } + + public int getInputHeight() { + return inputHeight; + } + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static ObbDetModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (ObbDetModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/exception/ObbDetException.java b/vision/src/main/java/cn/smartjavaai/obb/exception/ObbDetException.java new file mode 100644 index 0000000..e3e3a31 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/exception/ObbDetException.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.obb.exception; + +/** + * 旋转框检测异常 + * @author dwj + */ +public class ObbDetException extends RuntimeException{ + + public ObbDetException() { + super(); + } + + public ObbDetException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ObbDetException(String message, Throwable cause) { + super(message, cause); + } + + public ObbDetException(String message) { + super(message); + } + + public ObbDetException(Throwable cause) { + super(cause); + } + +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/model/CommonObbDetModel.java b/vision/src/main/java/cn/smartjavaai/obb/model/CommonObbDetModel.java new file mode 100644 index 0000000..3201935 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/model/CommonObbDetModel.java @@ -0,0 +1,160 @@ +package cn.smartjavaai.obb.model; + +import ai.djl.MalformedModelException; +import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.obb.config.ObbDetModelConfig; +import cn.smartjavaai.obb.criteria.ObbDetCriteriaFactory; +import cn.smartjavaai.obb.entity.ObbResult; +import cn.smartjavaai.obb.exception.ObbDetException; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.vision.utils.DetectedObjectsFilter; +import cn.smartjavaai.vision.utils.DetectorUtils; +import cn.smartjavaai.vision.utils.ObbResultFilter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.opencv.core.Mat; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 旋转框模型 + * @author dwj + */ +@Slf4j +public class CommonObbDetModel implements ObbDetModel { + + + private ObbDetModelConfig config; + + private ZooModel model; + + private GenericObjectPool> predictorPool; + + @Override + public void loadModel(ObbDetModelConfig config) { + if(Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型枚举"); + } + Criteria criteria = ObbDetCriteriaFactory.createCriteria(config); + this.config = config; + try { + model = criteria.loadModel(); + // 创建池子：每个线程独享 Predictor + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + int predictorPoolSize = config.getPredictorPoolSize(); + if(config.getPredictorPoolSize() <= 0){ + predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数 + } + predictorPool.setMaxTotal(predictorPoolSize); + log.debug("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前引擎: " + Engine.getInstance().getEngineName()); + log.debug("模型推理器线程池最大数量: " + predictorPoolSize); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new DetectionException("模型加载失败", e); + } + } + + @Override + public R detect(Image image) { + ObbResult obbResult = detectCore(image); + DetectionResponse detectionResponse = DetectorUtils.obbToToDetectionResponse(obbResult); + return R.ok(detectionResponse); + } + + /** + * 模型核心推理方法 + * @param image + * @return + */ + @Override + public ObbResult detectCore(Image image) { + Predictor predictor = null; + try { + predictor = predictorPool.borrowObject(); + ObbResult obbResult = predictor.predict(image); + //过滤 + if(Objects.nonNull(obbResult) && CollectionUtils.isNotEmpty(obbResult.getRotatedBoxeList())){ + ObbResultFilter obbResultFilter = new ObbResultFilter(config.getAllowedClasses(), config.getTopK()); + obbResult = obbResultFilter.filter(obbResult); + } + return obbResult; + } catch (Exception e) { + throw new DetectionException("旋转框错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + log.debug("释放资源"); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + } + + @Override + public R detectAndDraw(Image image) { + ObbResult obbResult = detectCore(image); + if(Objects.isNull(obbResult) || CollectionUtils.isEmpty(obbResult.getRotatedBoxeList())){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } + DetectorUtils.drawRectWithText(image, obbResult.getRotatedBoxeList()); + DetectionResponse detectionResponse = DetectorUtils.obbToToDetectionResponse(obbResult); + detectionResponse.setDrawnImage(image); + return R.ok(detectionResponse); + } + + @Override + public R detectAndDraw(String imagePath, String outputPath) { + try { + Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); + ObbResult obbResult = detectCore(img); + if(Objects.isNull(obbResult) || CollectionUtils.isEmpty(obbResult.getRotatedBoxeList())){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } + DetectorUtils.drawRectWithText(img, obbResult.getRotatedBoxeList()); + img.save(Files.newOutputStream(Paths.get(outputPath)), "png"); + DetectionResponse detectionResponse = DetectorUtils.obbToToDetectionResponse(obbResult); + return R.ok(detectionResponse); + } catch (IOException e) { + throw new ObbDetException(e); + } + } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModel.java b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModel.java new file mode 100644 index 0000000..f455fce --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModel.java @@ -0,0 +1,60 @@ +package cn.smartjavaai.obb.model; + +import ai.djl.modality.cv.Image; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.obb.config.ObbDetModelConfig; +import cn.smartjavaai.obb.entity.ObbResult; + +/** + * 旋转框检测模型 + * @author dwj + */ +public interface ObbDetModel extends AutoCloseable{ + + + /** + * 加载模型 + * @param config + */ + void loadModel(ObbDetModelConfig config); + + /** + * 旋转框 + * @param image + * @return + */ + default R detect(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 旋转框检测 核心方法 + * @param image + * @return + */ + default ObbResult detectCore(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 旋转框检测并绘制 + * @param image + * @return + */ + default R detectAndDraw(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 旋转框检测并绘制 + * @param imagePath + * @param outputPath + * @return + */ + default R detectAndDraw(String imagePath, String outputPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java new file mode 100644 index 0000000..c375f72 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java @@ -0,0 +1,107 @@ +package cn.smartjavaai.obb.model; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.obb.config.ObbDetModelConfig; +import cn.smartjavaai.obb.enums.ObbDetModelEnum; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 旋转框检测 模型工厂 + * @author dwj + */ +@Slf4j +public class ObbDetModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile ObbDetModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private ObbDetModelFactory() {} + + // 双重检查锁定的单例方法 + public static ObbDetModelFactory getInstance() { + if (instance == null) { + synchronized (ObbDetModelFactory.class) { + if (instance == null) { + instance = new ObbDetModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public ObbDetModel getModel(ObbDetModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private ObbDetModel createFaceDetModel(ObbDetModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + ObbDetModel model = null; + try { + model = (ObbDetModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(ObbDetModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(ObbDetModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(ObbDetModelEnum.YOLOV11, CommonObbDetModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/obb/translator/YoloV11OddTranslator.java b/vision/src/main/java/cn/smartjavaai/obb/translator/YoloV11OddTranslator.java new file mode 100644 index 0000000..b72ed29 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/obb/translator/YoloV11OddTranslator.java @@ -0,0 +1,532 @@ +package cn.smartjavaai.obb.translator; + +import ai.djl.Model; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.transform.*; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import ai.djl.translate.*; +import ai.djl.util.Utils; +import cn.smartjavaai.common.utils.LetterBoxUtils; +import cn.smartjavaai.obb.entity.ObbResult; +import cn.smartjavaai.obb.entity.YoloRotatedBox; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; + + +/** + * YoloV11OddTranslator + * @author dwj + */ +public class YoloV11OddTranslator implements Translator { + + private int maxBoxes; + + private YoloOutputType yoloOutputLayerType; + private float nmsThreshold; + + protected float threshold; + + protected List classes; + protected boolean applyRatio; + protected boolean removePadding; + + protected Pipeline pipeline; + private Image.Flag flag; + private Batchifier batchifier; + + protected int width; + protected int height; + + private SynsetLoader synsetLoader; + + + + @Override + public void prepare(TranslatorContext ctx) throws IOException { + if (this.classes == null) { + this.classes = this.synsetLoader.load(ctx.getModel()); + } + } + + + + + /** + * Constructs an ImageTranslator with the provided builder. + * + * @param builder the data to build with + */ + protected YoloV11OddTranslator(Builder builder) { + this.yoloOutputLayerType = builder.outputType; + this.nmsThreshold = builder.nmsThreshold; + maxBoxes = builder.maxBox; + this.threshold = builder.threshold; + this.synsetLoader = builder.synsetLoader; + this.applyRatio = builder.applyRatio; + this.removePadding = builder.removePadding; + this.flag = builder.flag; + this.pipeline = builder.pipeline; + this.batchifier = builder.batchifier; + this.width = builder.width; + this.height = builder.height; + } + + /** + * Creates a builder to build a {@code YoloV8Translator} with specified arguments. + * + * @return a new builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Creates a builder to build a {@code YoloV8Translator} with specified arguments. + * + * @param arguments arguments to specify builder options + * @return a new builder + */ + public static Builder builder(Map arguments) { + Builder builder = new Builder(); + builder.configPreProcess(arguments); + builder.configPostProcess(arguments); + return builder; + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + NDManager manager = ctx.getNDManager(); + NDArray array = input.toNDArray(manager, Image.Flag.COLOR); + int imageWidth = (int) array.getShape().get(1); + int imageHeight = (int) array.getShape().get(0); + //Letter box resize 640x640 with padding (保持比例，补边缘) + LetterBoxUtils.ResizeResult letterBoxResult = LetterBoxUtils.letterbox(manager, array, width, height, 114f, LetterBoxUtils.PaddingPosition.CENTER); + array = letterBoxResult.image; + // 转为 float32 且归一化到 0~1 + array = array.toType(DataType.FLOAT32, false).div(255f); // HWC + // HWC -> CHW + array = array.transpose(2, 0, 1); // CHW + + ctx.setAttachment("width", input.getWidth()); + ctx.setAttachment("height", input.getHeight()); + ctx.setAttachment("processedWidth", width); + ctx.setAttachment("processedHeight", height); + return new NDList(array); + } + + + /** {@inheritDoc} */ + protected ObbResult processFromBoxOutput(int imageWidth, int imageHeight, int processedWidth, int processedHeight, NDList list) { + + + float scale = Math.min((float) processedWidth / imageWidth, (float) processedHeight / imageHeight); + float padW = (processedWidth - imageWidth * scale) / 2; + float padH = (processedHeight - imageHeight * scale) / 2; + //[cx,cy,w,h,class*15,rotated] + NDArray rawResult = list.get(0); + NDArray reshapedResult = rawResult.transpose(); + Shape shape = reshapedResult.getShape(); + float[] buf = reshapedResult.toFloatArray(); + int numberRows = Math.toIntExact(shape.get(0)); + int nClasses = Math.toIntExact(shape.get(1)); + + // reverse order search in heap; searches through #maxBoxes for optimization when set + List rotatedBoxes = new ArrayList<>(); + for (int i = numberRows - 1; i > numberRows - maxBoxes; --i) { + int index = i * nClasses; + + // 找最大类别 + float maxClassProb = -1f; + int maxIndex = -1; + for (int c = 4; c < nClasses - 1; c++) { // 类别从4开始，-1是rotated + float classProb = buf[index + c]; + if (classProb > maxClassProb) { + maxClassProb = classProb; + maxIndex = c - 4; + } + } + + if (maxClassProb > threshold) { + float cx = buf[index]; + float cy = buf[index + 1]; + float w = buf[index + 2]; + float h = buf[index + 3]; + + cx = (cx - padW) / scale; + cy = (cy - padH) / scale; + w = w / scale; + h = h / scale; + float angle = buf[index + nClasses - 1]; // 最后一个是旋转角度 + YoloRotatedBox rotatedBox = new YoloRotatedBox(cx, cy, w, h, angle, classes.get(maxIndex), maxClassProb); + rotatedBoxes.add(rotatedBox); + } + } + List rotatedBoxeList = rotatedNMS(rotatedBoxes, nmsThreshold); + return new ObbResult(rotatedBoxeList); + } + + + public static List rotatedNMS(List boxes, double iouThreshold) { + List keep = new ArrayList<>(); + boolean[] removed = new boolean[boxes.size()]; + + // 按 score 降序 +// boxes.sort((b1, b2) -> Float.compare(b2.score, b1.score)); + + for (int i = 0; i < boxes.size(); i++) { + if (removed[i]) continue; + YoloRotatedBox ibox = boxes.get(i); + keep.add(ibox); + + for (int j = i + 1; j < boxes.size(); j++) { + if (removed[j]) continue; + YoloRotatedBox jbox = boxes.get(j); + + if (!ibox.className.equals(jbox.className)) continue; + + double iou = YoloRotatedBox.probiou(ibox, jbox, 1e-7); + if (iou > iouThreshold - 1e-7) { + removed[j] = true; + } + } + } + return keep; + } + + private ObbResult processFromDetectOutput() { + throw new UnsupportedOperationException( + "detect layer output is not supported yet, check correct YoloV5 export format"); + } + + @Override + public ObbResult processOutput(TranslatorContext ctx, NDList list) throws Exception { + int imageWidth = (Integer) ctx.getAttachment("width"); + int imageHeight = (Integer) ctx.getAttachment("height"); + int processedWidth = (Integer) ctx.getAttachment("processedWidth"); + int processedHeight = (Integer) ctx.getAttachment("processedHeight"); + + switch (yoloOutputLayerType) { + case DETECT: + return processFromDetectOutput(); + case AUTO: + if (list.get(0).getShape().dimension() > 2) { + return processFromDetectOutput(); + } else { + return processFromBoxOutput(imageWidth, imageHeight, processedWidth, processedHeight, list); + } + case BOX: + default: + return processFromBoxOutput(imageWidth, imageHeight, processedWidth, processedHeight, list); + } + } + + + + public static class Builder { + + private int maxBox = 8400; + + YoloOutputType outputType; + float nmsThreshold; + + protected float threshold = 0.25F; + protected boolean applyRatio; + protected boolean removePadding; + + protected int width = 224; + protected int height = 224; + protected Image.Flag flag; + protected Pipeline pipeline; + protected Batchifier batchifier; + + protected SynsetLoader synsetLoader; + + public Builder() { + this.outputType = YoloOutputType.AUTO; + this.nmsThreshold = 0.45F; + } + + public Builder optOutputType(YoloOutputType outputType) { + this.outputType = outputType; + return this; + } + + public Builder optNmsThreshold(float nmsThreshold) { + this.nmsThreshold = nmsThreshold; + return this; + } + + /** + * Builds the translator. + * + * @return the new translator + */ + public YoloV11OddTranslator build() { + if (pipeline == null) { + addTransform( + array -> array.transpose(2, 0, 1).toType(DataType.FLOAT32, false).div(255)); + } +// validate(); + return new YoloV11OddTranslator(this); + } + + protected Builder self() { + return this; + } + + public Builder addTransform(Transform transform) { + if (this.pipeline == null) { + this.pipeline = new Pipeline(); + } + + this.pipeline.add(transform); + return this.self(); + } + + public Builder optApplyRatio(boolean value) { + this.applyRatio = value; + return this.self(); + } + + public Builder optFlag(Image.Flag flag) { + this.flag = flag; + return this.self(); + } + + public Builder setPipeline(Pipeline pipeline) { + this.pipeline = pipeline; + return this.self(); + } + + public Builder setImageSize(int width, int height) { + this.width = width; + this.height = height; + return this.self(); + } + + + public Builder optBatchifier(Batchifier batchifier) { + this.batchifier = batchifier; + return this.self(); + } + + public Builder optThreshold(float threshold) { + this.threshold = threshold; + return this.self(); + } + + /** + * Sets the name of the synset file listing the potential classes for an image. + * + * @param synsetArtifactName a file listing the potential classes for an image + * @return the builder + */ + public Builder optSynsetArtifactName(String synsetArtifactName) { + synsetLoader = new SynsetLoader(synsetArtifactName); + return self(); + } + + /** + * Sets the URL of the synset file. + * + * @param synsetUrl the URL of the synset file + * @return the builder + */ + public Builder optSynsetUrl(String synsetUrl) { + try { + this.synsetLoader = new SynsetLoader(new URL(synsetUrl)); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid synsetUrl: " + synsetUrl, e); + } + return self(); + } + + /** + * Sets the potential classes for an image. + * + * @param synset the potential classes for an image + * @return the builder + */ + public Builder optSynset(List synset) { + synsetLoader = new SynsetLoader(synset); + return self(); + } + + /** {@inheritDoc} */ + protected void configPostProcess(Map arguments) { + if (ArgumentsUtil.booleanValue(arguments, "optApplyRatio") || ArgumentsUtil.booleanValue(arguments, "applyRatio")) { + this.optApplyRatio(true); + } + this.threshold = ArgumentsUtil.floatValue(arguments, "threshold", 0.25F); + String centerFit = ArgumentsUtil.stringValue(arguments, "centerFit", "false"); + this.removePadding = "true".equals(centerFit); + String type = ArgumentsUtil.stringValue(arguments, "outputType", "AUTO"); + this.outputType = YoloOutputType.valueOf(type.toUpperCase(Locale.ENGLISH)); + this.nmsThreshold = ArgumentsUtil.floatValue(arguments, "nmsThreshold", 0.45F); + maxBox = ArgumentsUtil.intValue(arguments, "maxBox", 8400); + } + + protected void configPreProcess(Map arguments) { + if (this.pipeline == null) { + this.pipeline = new Pipeline(); + } + + this.width = ArgumentsUtil.intValue(arguments, "width", 224); + this.height = ArgumentsUtil.intValue(arguments, "height", 224); + if (arguments.containsKey("flag")) { + this.flag = Image.Flag.valueOf(arguments.get("flag").toString()); + } + + String pad = ArgumentsUtil.stringValue(arguments, "pad", "false"); + if ("true".equals(pad)) { + this.addTransform(new Pad(0.0)); + } else if (!"false".equals(pad)) { + double padding = Double.parseDouble(pad); + this.addTransform(new Pad(padding)); + } + + String resize = ArgumentsUtil.stringValue(arguments, "resize", "false"); + int w; + int shortEdge; + if ("true".equals(resize)) { + this.addTransform(new Resize(this.width, this.height)); + } else if (!"false".equals(resize)) { + String[] tokens = resize.split("\\s*,\\s*"); + w = (int)Double.parseDouble(tokens[0]); + if (tokens.length > 1) { + shortEdge = (int)Double.parseDouble(tokens[1]); + } else { + shortEdge = w; + } + + Image.Interpolation interpolation; + if (tokens.length > 2) { + interpolation = Image.Interpolation.valueOf(tokens[2]); + } else { + interpolation = Image.Interpolation.BILINEAR; + } + + this.addTransform(new Resize(w, shortEdge, interpolation)); + } + + String resizeShort = ArgumentsUtil.stringValue(arguments, "resizeShort", "false"); + if ("true".equals(resizeShort)) { + w = Math.max(this.width, this.height); + this.addTransform(new ResizeShort(w)); + } else if (!"false".equals(resizeShort)) { + String[] tokens = resizeShort.split("\\s*,\\s*"); + shortEdge = (int)Double.parseDouble(tokens[0]); + int longEdge; + if (tokens.length > 1) { + longEdge = (int)Double.parseDouble(tokens[1]); + } else { + longEdge = -1; + } + + Image.Interpolation interpolation; + if (tokens.length > 2) { + interpolation = Image.Interpolation.valueOf(tokens[2]); + } else { + interpolation = Image.Interpolation.BILINEAR; + } + + this.addTransform(new ResizeShort(shortEdge, longEdge, interpolation)); + } + + if (ArgumentsUtil.booleanValue(arguments, "centerCrop", false)) { + this.addTransform(new CenterCrop(this.width, this.height)); + } + + if (ArgumentsUtil.booleanValue(arguments, "centerFit")) { + this.addTransform(new CenterFit(this.width, this.height)); + } + + if (ArgumentsUtil.booleanValue(arguments, "toTensor", true)) { + this.addTransform(new ToTensor()); + } + + String normalize = ArgumentsUtil.stringValue(arguments, "normalize", "false"); + if ("true".equals(normalize)) { + float[] MEAN = new float[]{0.485F, 0.456F, 0.406F}; + float[] STD = new float[]{0.229F, 0.224F, 0.225F}; + this.addTransform(new Normalize(MEAN, STD)); + } else if (!"false".equals(normalize)) { + String[] tokens = normalize.split("\\s*,\\s*"); + if (tokens.length != 6) { + throw new IllegalArgumentException("Invalid normalize value: " + normalize); + } + + float[] mean = new float[]{Float.parseFloat(tokens[0]), Float.parseFloat(tokens[1]), Float.parseFloat(tokens[2])}; + float[] std = new float[]{Float.parseFloat(tokens[3]), Float.parseFloat(tokens[4]), Float.parseFloat(tokens[5])}; + this.addTransform(new Normalize(mean, std)); + } + + String range = (String)arguments.get("range"); + if ("0,1".equals(range)) { + this.addTransform((a) -> { + return a.div(255.0F); + }); + } else if ("-1,1".equals(range)) { + this.addTransform((a) -> { + return a.div(128.0F).sub(1); + }); + } + + if (arguments.containsKey("batchifier")) { + this.batchifier = Batchifier.fromString((String)arguments.get("batchifier")); + } + + } + } + + public static enum YoloOutputType { + BOX, + DETECT, + AUTO; + + private YoloOutputType() { + } + } + + protected static final class SynsetLoader { + + private String synsetFileName; + private URL synsetUrl; + private List synset; + + public SynsetLoader(List synset) { + this.synset = synset; + } + + public SynsetLoader(URL synsetUrl) { + this.synsetUrl = synsetUrl; + } + + public SynsetLoader(String synsetFileName) { + this.synsetFileName = synsetFileName; + } + + public List load(Model model) throws IOException { + if (synset != null) { + return synset; + } else if (synsetUrl != null) { + try (InputStream is = synsetUrl.openStream()) { + return Utils.readLines(is); + } + } + return model.getArtifact(synsetFileName, Utils::readLines); + } + } + + + + +} diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/config/PersonDetModelConfig.java b/vision/src/main/java/cn/smartjavaai/objectdetection/config/PersonDetModelConfig.java index b3f3f5f..4b5f517 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/config/PersonDetModelConfig.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/config/PersonDetModelConfig.java @@ -34,12 +34,6 @@ public class PersonDetModelConfig extends ModelConfig { */ private String modelPath; - - /** - * 允许的分类列表 - */ - private List allowedClasses; - /** * 按置信度分数排序后，最多保留的检测框数量 */ diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/criteria/CriteriaBuilderFactory.java b/vision/src/main/java/cn/smartjavaai/objectdetection/criteria/CriteriaBuilderFactory.java index da8aca1..10a70ab 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/criteria/CriteriaBuilderFactory.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/criteria/CriteriaBuilderFactory.java @@ -17,23 +17,23 @@ public class CriteriaBuilderFactory { public static Criteria createCriteria(DetectorModelConfig config) { //以下模型modelPath不允许为空 - if(config.getModelEnum() == DetectorModelEnum.YOLOV8_OFFICIAL || - config.getModelEnum() == DetectorModelEnum.YOLOV12_OFFICIAL || - config.getModelEnum() == DetectorModelEnum.YOLOV8_CUSTOM || - config.getModelEnum() == DetectorModelEnum.YOLOV12_CUSTOM || + if(config.getModelEnum() == DetectorModelEnum.YOLOV8_OFFICIAL_ONNX || + config.getModelEnum() == DetectorModelEnum.YOLOV12_OFFICIAL_ONNX || + config.getModelEnum() == DetectorModelEnum.YOLOV8_CUSTOM_ONNX || + config.getModelEnum() == DetectorModelEnum.YOLOV12_CUSTOM_ONNX || config.getModelEnum() == DetectorModelEnum.TENSORFLOW2_OFFICIAL){ if(StringUtils.isBlank(config.getModelPath())){ throw new DetectionException("modelPath is null"); } } switch (config.getModelEnum()) { - case YOLOV8_OFFICIAL: + case YOLOV8_OFFICIAL_ONNX: return new YoloCriteriaBuilder().buildCriteria(config); - case YOLOV12_OFFICIAL: + case YOLOV12_OFFICIAL_ONNX: return new YoloCriteriaBuilder().buildCriteria(config); - case YOLOV8_CUSTOM: + case YOLOV8_CUSTOM_ONNX: return new YoloCriteriaBuilder().buildCriteria(config); - case YOLOV12_CUSTOM: + case YOLOV12_CUSTOM_ONNX: return new YoloCriteriaBuilder().buildCriteria(config); case TENSORFLOW2_OFFICIAL: return new Tensorflow2CriteriaBuilder().buildCriteria(config); diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/enums/DetectorModelEnum.java b/vision/src/main/java/cn/smartjavaai/objectdetection/enums/DetectorModelEnum.java index c03cc6c..1b3fa99 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/enums/DetectorModelEnum.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/enums/DetectorModelEnum.java @@ -8,40 +8,43 @@ package cn.smartjavaai.objectdetection.enums; public enum DetectorModelEnum { // resnet50 系列 - SSD_300_RESNET50("ai.djl.pytorch/ssd/0.0.1/ssd_300_resnet50"), - SSD_512_RESNET50_V1_VOC("ai.djl./ssd/0.0.1/ssd_512_resnet50_v1_voc"), + SSD_300_RESNET50_DJL("ai.djl.pytorch/ssd/0.0.1/ssd_300_resnet50"), + SSD_512_RESNET50_V1_VOC_DJL("ai.djl./ssd/0.0.1/ssd_512_resnet50_v1_voc"), // vgg16 系列 - SSD_512_VGG16_ATROUS_COCO("ai.djl.mxnet/ssd/0.0.1/ssd_512_vgg16_atrous_coco"), - SSD_300_VGG16_ATROUS_VOC("ai.djl.mxnet/ssd/0.0.1/ssd_300_vgg16_atrous_voc"), + SSD_512_VGG16_ATROUS_COCO_DJL("ai.djl.mxnet/ssd/0.0.1/ssd_512_vgg16_atrous_coco"), + SSD_300_VGG16_ATROUS_VOC_DJL("ai.djl.mxnet/ssd/0.0.1/ssd_300_vgg16_atrous_voc"), // mobilenet 系列 - SSD_512_MOBILENET1_VOC("ai.djl.mxnet/ssd/0.0.1/ssd_512_mobilenet1.0_voc"), + SSD_512_MOBILENET1_VOC_DJL("ai.djl.mxnet/ssd/0.0.1/ssd_512_mobilenet1.0_voc"), // YOLO 系列 - YOLOV8N("ai.djl.pytorch/yolov8n/0.0.1/yolov8n"), - YOLO11N("ai.djl.pytorch/yolo11n/0.0.1/yolo11n"), - YOLOV5S("ai.djl.pytorch/yolo5s/0.0.1/yolov5s"), - YOLOV5S_ONNXRUNTIME("ai.djl.onnxruntime/yolo5s/0.0.1/yolo5s"), - YOLO("ai.djl.mxnet/yolo/0.0.1/yolo"), +// YOLOV8N("ai.djl.pytorch/yolov8n/0.0.1/yolov8n"), +// YOLO11N("ai.djl.pytorch/yolo11n/0.0.1/yolo11n"), + YOLOV5S_DJL("ai.djl.pytorch/yolo5s/0.0.1/yolov5s"), + YOLOV5S_ONNX_DJL("ai.djl.onnxruntime/yolo5s/0.0.1/yolo5s"), + YOLO_DJL("ai.djl.mxnet/yolo/0.0.1/yolo"), // YOLOv3 变体 - YOLO3_DARKNET_VOC_416("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_voc_416"), - YOLO3_MOBILENET_VOC_320("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_voc_320"), - YOLO3_MOBILENET_VOC_416("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_voc_41"), - YOLO3_DARKNET_COCO_320("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_320"), - YOLO3_DARKNET_COCO_416("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_416"), - YOLO3_DARKNET_COCO_608("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_608"), - YOLO3_MOBILENET_COCO_320("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_320"), - YOLO3_MOBILENET_COCO_416("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_416"), - YOLO3_MOBILENET_COCO_608("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_608"), + YOLO3_DARKNET_VOC_416_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_voc_416"), + YOLO3_MOBILENET_VOC_320_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_voc_320"), + YOLO3_MOBILENET_VOC_416_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_voc_41"), + YOLO3_DARKNET_COCO_320_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_320"), + YOLO3_DARKNET_COCO_416_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_416"), + YOLO3_DARKNET_COCO_608_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_darknet_coco_608"), + YOLO3_MOBILENET_COCO_320_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_320"), + YOLO3_MOBILENET_COCO_416_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_416"), + YOLO3_MOBILENET_COCO_608_DJL("ai.djl.mxnet/yolo/0.0.1/yolo3_mobilenet_coco_608"), - YOLOV12_OFFICIAL(""), - YOLOV8_OFFICIAL(""), - YOLOV8_CUSTOM(""), + YOLOV8_OFFICIAL_ONNX(""), + YOLOV11_OFFICIAL_ONNX(""), + YOLOV12_OFFICIAL_ONNX(""), - YOLOV12_CUSTOM(""), + + YOLOV8_CUSTOM_ONNX(""), + YOLOV11_CUSTOM_ONNX(""), + YOLOV12_CUSTOM_ONNX(""), // TensorFlow 2.x 官方模型 TENSORFLOW2_OFFICIAL(""); diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java b/vision/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java index fb16e38..4b55bf4 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java @@ -11,9 +11,9 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.pool.PredictorFactory; import cn.smartjavaai.common.utils.FileUtils; -import cn.smartjavaai.common.utils.FrameConverterUtil; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.common.utils.OpenCVUtils; import cn.smartjavaai.objectdetection.config.DetectorModelConfig; @@ -119,6 +119,9 @@ public class DetectorModel implements AutoCloseable{ try { img = ImageFactory.getInstance().fromFile(Paths.get(imagePath)); DetectedObjects detectedObjects = detect(img); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + throw new DetectionException("未检测到图片中的物体"); + } img.drawBoundingBoxes(detectedObjects); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); // 调用 save 方法将 Image 写入字节流 @@ -186,6 +189,9 @@ public class DetectorModel implements AutoCloseable{ } Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage)); DetectedObjects detectedObjects = detect(img); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + throw new DetectionException("未检测到图片中的物体"); + } img.drawBoundingBoxes(detectedObjects); try { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); @@ -225,7 +231,6 @@ public class DetectorModel implements AutoCloseable{ if (predictor != null) { try { predictorPool.returnObject(predictor); //归还 - log.debug("释放资源"); } catch (Exception e) { log.warn("归还Predictor失败", e); try { diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/model/ObjectDetectionModelFactory.java b/vision/src/main/java/cn/smartjavaai/objectdetection/model/ObjectDetectionModelFactory.java index 8122587..e1ef197 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/model/ObjectDetectionModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/model/ObjectDetectionModelFactory.java @@ -61,12 +61,12 @@ public class ObjectDetectionModelFactory { * 获取默认模型 * @return */ - public DetectorModel getModel() { - // 初始化默认配置 - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.YOLO11N); - return getModel(config); - } +// public DetectorModel getModel() { +// // 初始化默认配置 +// DetectorModelConfig config = new DetectorModelConfig(); +// config.setModelEnum(DetectorModelEnum.YOLO11N); +// return getModel(config); +// } /** * 关闭所有已加载的模型 diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/CommonPersonDetModel.java b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/CommonPersonDetModel.java index 1fa7347..c03d138 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/CommonPersonDetModel.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/CommonPersonDetModel.java @@ -8,10 +8,12 @@ import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; +import cn.hutool.core.img.ImgUtil; import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.obb.config.ObbDetModelConfig; import cn.smartjavaai.obb.criteria.ObbDetCriteriaFactory; import cn.smartjavaai.obb.entity.ObbResult; @@ -27,6 +29,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.pool2.impl.GenericObjectPool; import org.opencv.core.Mat; +import java.awt.image.BufferedImage; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -92,7 +95,7 @@ public class CommonPersonDetModel implements PersonDetModel { DetectedObjects detectedObjects = predictor.predict(image); //过滤 if(Objects.nonNull(detectedObjects) && detectedObjects.getNumberOfObjects() > 0){ - DetectedObjectsFilter detectedObjectsFilter = new DetectedObjectsFilter(config.getAllowedClasses(), config.getTopK()); + DetectedObjectsFilter detectedObjectsFilter = new DetectedObjectsFilter(null, config.getTopK()); detectedObjects = detectedObjectsFilter.filter(detectedObjects); } return detectedObjects; @@ -118,9 +121,13 @@ public class CommonPersonDetModel implements PersonDetModel { @Override public R detectAndDraw(Image image) { DetectedObjects detectedObjects = detectCore(image); - image.drawBoundingBoxes(detectedObjects); - DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image); - detectionResponse.setDrawnImage(image); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } + Image drawnImage = ImageUtils.copy(image); + drawnImage.drawBoundingBoxes(detectedObjects); + DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, drawnImage); + detectionResponse.setDrawnImage(drawnImage); return R.ok(detectionResponse); } @@ -129,6 +136,9 @@ public class CommonPersonDetModel implements PersonDetModel { try { Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); DetectedObjects detectedObjects = detectCore(img); + if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ + return R.fail(R.Status.NO_OBJECT_DETECTED); + } img.drawBoundingBoxes(detectedObjects); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); // 调用 save 方法将 Image 写入字节流 diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java new file mode 100644 index 0000000..d480cba --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java @@ -0,0 +1,109 @@ +package cn.smartjavaai.objectdetection.model.person; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.objectdetection.config.PersonDetModelConfig; +import cn.smartjavaai.objectdetection.config.PersonDetModelConfig; +import cn.smartjavaai.objectdetection.enums.PersonDetectorModelEnum; +import cn.smartjavaai.objectdetection.enums.PersonDetectorModelEnum; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 行人检测 模型工厂 + * @author dwj + */ +@Slf4j +public class PersonDetModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile PersonDetModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private PersonDetModelFactory() {} + + // 双重检查锁定的单例方法 + public static PersonDetModelFactory getInstance() { + if (instance == null) { + synchronized (PersonDetModelFactory.class) { + if (instance == null) { + instance = new PersonDetModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public PersonDetModel getModel(PersonDetModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private PersonDetModel createFaceDetModel(PersonDetModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + PersonDetModel model = null; + try { + model = (PersonDetModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(PersonDetectorModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(PersonDetectorModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(PersonDetectorModelEnum.YOLOV8_PERSON, CommonPersonDetModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetectionListener.java b/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetectionListener.java index 6a80f0f..a839bfb 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetectionListener.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetectionListener.java @@ -12,5 +12,21 @@ import java.util.List; public interface StreamDetectionListener { + + /** + * 当检测到目标时回调 + * @param detectionInfoList 目标信息列表 + * @param image 检测到的图片 + */ void onObjectDetected(List detectionInfoList, Image image); + + /** + * 当视频文件读取完毕时回调 + */ + void onStreamEnded(); + + /** + * 当视频流断开连接时回调 + */ + void onStreamDisconnected(); } diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetector.java b/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetector.java index 4532422..604ae76 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetector.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/stream/StreamDetector.java @@ -1,15 +1,16 @@ package cn.smartjavaai.objectdetection.stream; +import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; import ai.djl.modality.cv.output.BoundingBox; import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.modality.cv.output.Mask; import ai.djl.modality.cv.output.Rectangle; +import cn.hutool.core.date.LocalDateTimeUtil; import cn.hutool.core.lang.UUID; import cn.smartjavaai.common.entity.*; import cn.smartjavaai.common.enums.VideoSourceType; -import cn.smartjavaai.common.utils.FrameConverterUtil; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.common.utils.OpenCVUtils; import cn.smartjavaai.objectdetection.exception.DetectionException; @@ -18,11 +19,10 @@ import cn.smartjavaai.vision.utils.DetectorUtils; import lombok.extern.slf4j.Slf4j; import nu.pattern.OpenCV; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacv.*; -import org.bytedeco.opencv.global.opencv_imgcodecs; -import org.opencv.core.Core; import org.opencv.core.Mat; -import org.opencv.imgcodecs.Imgcodecs; import java.util.*; import java.util.concurrent.*; @@ -41,10 +41,14 @@ public class StreamDetector implements AutoCloseable{ private DetectorModel detectorModel; private String streamUrl; - private ExecutorService grabberExecutor; // 专门抓帧的线程 - private ExecutorService processorExecutor; // 专门处理帧的线程 + //专门抓帧的线程 + private ExecutorService grabberExecutor; + //专门处理帧的线程 + private ExecutorService processorExecutor; + //回调线程池 + ExecutorService callbackExecutor; private int frameDetectionInterval = 1; - private int repeatGap = 5; // 秒 + private long repeatGap = 5; // 秒 private volatile boolean isRunning; private FrameGrabber grabber; private StreamDetectionListener listener; @@ -55,6 +59,18 @@ public class StreamDetector implements AutoCloseable{ private Map lastDetectTime = new ConcurrentHashMap<>(); private BlockingQueue frameQueue = new LinkedBlockingQueue<>(100); + private GenericObjectPool> predictorPool; + + private Predictor predictor; + + boolean grabberFinished = false; // 标记结束 + + //空帧数量 + private int nullFrameCount = 0; + + // 连续多少次空帧认为断联 + private static final int MAX_NULL_FRAMES = 5; + public static Builder builder() { return new Builder(); } private StreamDetector(Builder builder) { @@ -64,9 +80,8 @@ public class StreamDetector implements AutoCloseable{ this.listener = builder.listener; this.sourceType = builder.sourceType; this.cameraIndex = builder.cameraIndex; + this.repeatGap = builder.repeatGap; this.converterToMat = new OpenCVFrameConverter.ToOrgOpenCvCoreMat(); - - } private void initializeGrabber() throws FrameGrabber.Exception { @@ -78,34 +93,61 @@ public class StreamDetector implements AutoCloseable{ if (sourceType == VideoSourceType.STREAM) { grabber.setOption("rtsp_transport", "tcp"); grabber.setOption("buffer_size", "1024000"); - grabber.setOption("stimeout", "20000000"); - grabber.setOption("max_delay", "500000"); + grabber.setOption("stimeout", "2000000"); // 超时：单位微秒，这里是2秒 + grabber.setOption("rw_timeout", "2000000"); // 读超时 + grabber.setOption("max_delay", "5000000"); + grabber.setOption("timeout", "2000000"); // 总超时 } } + //日志级别 + avutil.av_log_set_level(avutil.AV_LOG_ERROR); grabber.start(); + if(sourceType == VideoSourceType.FILE){ + // 总帧数 + int totalFrames = grabber.getLengthInFrames(); + log.info("视频帧数：{}", totalFrames); + } } public void startDetection() { - if (isRunning) return; - isRunning = true; + if (isRunning){ + throw new RuntimeException("当前正在运行中"); + } + grabberFinished = false; + //获取模型Predictor + predictorPool = detectorModel.getPool(); + try { + predictor = predictorPool.borrowObject(); + } catch (Exception e) { + throw new DetectionException(e); + } // 初始化抓帧线程池 - if (grabberExecutor == null) grabberExecutor = Executors.newSingleThreadExecutor(); + if (grabberExecutor == null || grabberExecutor.isShutdown()) + grabberExecutor = Executors.newFixedThreadPool(2); // 初始化帧处理线程池 - if (processorExecutor == null) processorExecutor = Executors.newSingleThreadExecutor(); - + if (processorExecutor == null || processorExecutor.isShutdown()) + processorExecutor = Executors.newFixedThreadPool(2); + if (callbackExecutor == null || callbackExecutor.isShutdown()) + callbackExecutor = Executors.newFixedThreadPool(4); + try { + initializeGrabber(); + } catch (FrameGrabber.Exception e) { + throw new DetectionException("视频流检测启动失败", e); + } + isRunning = true; + log.debug("视频流处理已启动"); // 初始化抓帧线程 grabberExecutor.submit(() -> { try { - initializeGrabber(); processFrames(); } catch (Exception e) { log.error("视频流处理异常", e); } finally { - release(); + //标识抓取已结束 + grabberFinished = true; } }); - log.info("视频流处理已启动"); // 初始化队列处理线程：解决回调比较耗时，导致线程池爆满 startFrameProcessor(); } @@ -114,22 +156,42 @@ public class StreamDetector implements AutoCloseable{ * 负责抓取视频帧到队列 */ private void processFrames() { - int frameCount = 0; - while (isRunning) { + long frameCount = 0; + while (!grabberFinished && isRunning) { try { - Frame frame = grabber.grab(); - if (frame == null || frame.image == null) continue; - + Frame frame = grabber.grabFrame(); + if (frame == null || frame.image == null) { + if(sourceType == VideoSourceType.FILE){ + log.debug("视频检测结束"); + grabberFinished = true; + break; + }else{ + log.debug("未检测到视频帧"); + nullFrameCount++; + if (nullFrameCount > MAX_NULL_FRAMES) { + log.warn("检测到视频断开，已超过最大空帧次数"); + if(isRunning){ + stopDetection(); + } + if (listener != null) { + listener.onStreamDisconnected(); + } + break; + } + continue; + } + } + nullFrameCount = 0; // 只要拿到正常帧就清零 frameCount++; if (frameCount % frameDetectionInterval != 0) continue; - Frame currentFrame = frame.clone(); - frameQueue.offer(currentFrame); // 队列满则丢弃，可改为 put 阻塞 + frameQueue.offer(currentFrame); +// log.debug("正在抓取第{}帧，当前帧数：{}", frameCount, frameQueue.size()); } catch (Exception e) { log.error("抓取视频帧异常", e); - if (e instanceof FFmpegFrameGrabber.Exception) reconnect(); } } + log.debug("抓取帧线程退出"); } /** @@ -137,18 +199,29 @@ public class StreamDetector implements AutoCloseable{ */ private void startFrameProcessor() { processorExecutor.submit(() -> { - log.info("帧处理线程已启动"); - while (isRunning || !frameQueue.isEmpty()) { + log.debug("帧处理线程已启动"); + while ((!grabberFinished || !frameQueue.isEmpty()) && isRunning) { try { Frame frame = frameQueue.poll(100, TimeUnit.MILLISECONDS); - if (frame != null) processFrame(frame); + if (frame != null) { + processFrame(frame); + } } catch (InterruptedException e) { - e.printStackTrace(); + log.debug("帧处理线程被中断，准备退出"); + break; } catch (Exception e) { log.error("帧处理异常", e); } } + if(isRunning){ + stopDetection(); + } + if (listener != null) { + listener.onStreamEnded(); + } + log.debug("帧处理线程退出"); }); + } private void processFrame(Frame frame) { @@ -158,12 +231,16 @@ public class StreamDetector implements AutoCloseable{ if (mat == null) return; Image image = ImageFactory.getInstance().fromImage(mat); - DetectedObjects detectedObjects = detectorModel.detect(image); -// log.debug("检测结果：{}", detectedObjects.toString()); + DetectedObjects detectedObjects = predictor.predict(image); +// log.info("内部检测结果：{}", detectedObjects.toString()); DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image); + if(Objects.isNull(detectionResponse)){ + return; + } List filtered = filterRepeatedObjects(detectionResponse); if (!filtered.isEmpty() && listener != null) { - listener.onObjectDetected(filtered, image); // 同帧多物体一次回调 + Image copyImage = image.duplicate(); + callbackExecutor.submit(() -> listener.onObjectDetected(filtered, copyImage)); } } catch (Throwable e) { e.printStackTrace(); @@ -187,47 +264,86 @@ public class StreamDetector implements AutoCloseable{ return result; } - private void reconnect() { - log.info("尝试重新连接视频流"); - try { - release(); - Thread.sleep(5000); - initializeGrabber(); - } catch (Exception e) { - log.error("重新连接RTSP流失败", e); + + /** + * 开始检测下一个视频文件 + */ + public void startNextVideo(String videoPath) { + if(!grabberFinished){ + throw new DetectionException("当前视频未检测结束，请先关闭当前检测，再切换下一个视频"); } + if(sourceType != VideoSourceType.FILE){ + throw new DetectionException("sourceType不是文件"); + } + this.streamUrl = videoPath; + this.grabberFinished = false; + startDetection(); } - public void stopDetection() { isRunning = false; } - - private void release() { + public void stopDetection() { + log.debug("停止检测中..."); + isRunning = false; + grabberFinished = true; + if (predictor != null && predictorPool != null) { + try { + predictorPool.returnObject(predictor); //归还 + predictor = null; + predictorPool = null; + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } if (grabber != null) { - try { grabber.stop(); grabber.release(); } - catch (FrameGrabber.Exception e) { log.error("释放Grabber失败", e); } + try { + grabber.stop(); grabber.release(); + grabber = null; + }catch (FrameGrabber.Exception e) { + log.error("释放Grabber失败", e); + } } + log.debug("停止检测完毕"); } + @Override public void close() { - stopDetection(); - if (grabberExecutor != null) grabberExecutor.shutdownNow(); - if (processorExecutor != null) processorExecutor.shutdownNow(); - release(); +// if(isRunning){ +// System.out.println("--isRunning：" + isRunning); +// stopDetection(); +// } + if (grabberExecutor != null){ + grabberExecutor.shutdownNow(); + } + if (processorExecutor != null) { + processorExecutor.shutdownNow(); + } + if (callbackExecutor != null) { + callbackExecutor.shutdownNow(); + } } public static class Builder { private DetectorModel detectorModel; private String streamUrl; - private ExecutorService executorService; private int frameDetectionInterval = 1; private StreamDetectionListener listener; private VideoSourceType sourceType = VideoSourceType.STREAM; // 默认流 private int cameraIndex = 0; // 默认第一个摄像头 + private long repeatGap = 5;//同物体重复检测间隔 + public Builder detectorModel(DetectorModel m) { this.detectorModel = m; return this; } public Builder streamUrl(String url) { this.streamUrl = url; return this; } - public Builder executorService(ExecutorService es) { this.executorService = es; return this; } public Builder listener(StreamDetectionListener listener) { this.listener = listener; return this; } + public Builder repeatGap(long repeatGap) { + this.repeatGap = repeatGap; + return this; + } public Builder sourceType(VideoSourceType sourceType) { this.sourceType = sourceType; return this; @@ -268,10 +384,6 @@ public class StreamDetector implements AutoCloseable{ throw new DetectionException("不支持的视频源类型: " + sourceType); } - if (executorService == null) { - executorService = Executors.newFixedThreadPool(2); // 至少2个线程 - } - return new StreamDetector(this); } diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/translator/YoloV8PersonDetTranslator.java b/vision/src/main/java/cn/smartjavaai/objectdetection/translator/YoloV8PersonDetTranslator.java index 3fe5393..acda7a2 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/translator/YoloV8PersonDetTranslator.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/translator/YoloV8PersonDetTranslator.java @@ -136,7 +136,6 @@ public class YoloV8PersonDetTranslator implements Translator createCriteria(PoseModelConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId()); + } + Criteria criteria = null; + //DJL官方提供模型 + if(StringUtils.isNotBlank(config.getModelEnum().getModelUri())){ + criteria = createDJLCriteria(config, device); + } + return criteria; + } + + /** + * 创建DJL官方模型Criteria + * 需要模型同目录下存在：serving.properties + * @param config + * @return + */ + public static Criteria createDJLCriteria(PoseModelConfig config, Device device) { + if(StringUtils.isNotBlank(config.getModelPath()) + && DJLCommonUtils.isServingPropertiesExists(Paths.get(config.getModelPath()))){ + throw new PoseException("模型所在目录未找到 serving.properties 文件"); + } + Criteria criteria = + Criteria.builder() + .setTypes(Image.class, Joints[].class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : config.getModelEnum().getModelUri()) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optDevice(device) + .optArgument("threshold", config.getThreshold() > 0 ? config.getThreshold() : null) + .optProgress(new ProgressBar()) + .build(); + return criteria; + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/pose/enums/PoseModelEnum.java b/vision/src/main/java/cn/smartjavaai/pose/enums/PoseModelEnum.java new file mode 100644 index 0000000..6c20a33 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/pose/enums/PoseModelEnum.java @@ -0,0 +1,42 @@ +package cn.smartjavaai.pose.enums; + +/** + * 姿态估计模型枚举 + * @author dwj + */ +public enum PoseModelEnum { + + YOLO11N_POSE_PT("djl://ai.djl.pytorch/yolo11n-pose"), + + YOLOV8N_POSE_PT("djl://ai.djl.pytorch/yolov8n-pose"), + YOLO11N_POSE_ONNX("djl://ai.djl.onnxruntime/yolo11n-pose"), + + YOLOV8N_POSE_ONNX("djl://ai.djl.onnxruntime/yolov8n-pose"); + +// SIMPLE_POSE_MXNET("djl://ai.djl.mxnet/simple_pose"); + + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static PoseModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (PoseModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + private final String modelUri; + + PoseModelEnum(String modelUri) { + this.modelUri = modelUri; + } + + public String getModelUri() { + return modelUri; + } + +} diff --git a/vision/src/main/java/cn/smartjavaai/pose/exception/PoseException.java b/vision/src/main/java/cn/smartjavaai/pose/exception/PoseException.java new file mode 100644 index 0000000..236e142 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/pose/exception/PoseException.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.pose.exception; + +/** + * 动作检测异常 + * @author dwj + */ +public class PoseException extends RuntimeException{ + + public PoseException() { + super(); + } + + public PoseException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public PoseException(String message, Throwable cause) { + super(message, cause); + } + + public PoseException(String message) { + super(message); + } + + public PoseException(Throwable cause) { + super(cause); + } + +} diff --git a/vision/src/main/java/cn/smartjavaai/pose/model/CommonPoseModel.java b/vision/src/main/java/cn/smartjavaai/pose/model/CommonPoseModel.java new file mode 100644 index 0000000..463897d --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/pose/model/CommonPoseModel.java @@ -0,0 +1,147 @@ +package cn.smartjavaai.pose.model; + +import ai.djl.MalformedModelException; +import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.Joints; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.obb.exception.ObbDetException; +import cn.smartjavaai.objectdetection.config.PersonDetModelConfig; +import cn.smartjavaai.objectdetection.criteria.PersonDetCriteriaFactory; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.pose.config.PoseModelConfig; +import cn.smartjavaai.pose.criteria.PoseCriteriaFactory; +import cn.smartjavaai.vision.utils.DetectedObjectsFilter; +import cn.smartjavaai.vision.utils.DetectorUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.pool2.impl.GenericObjectPool; + +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 姿态估计模型 + * @author dwj + */ +@Slf4j +public class CommonPoseModel implements PoseModel { + + + private PoseModelConfig config; + + private ZooModel model; + + private GenericObjectPool> predictorPool; + + @Override + public void loadModel(PoseModelConfig config) { + if(Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型枚举"); + } + Criteria criteria = PoseCriteriaFactory.createCriteria(config); + this.config = config; + try { + model = criteria.loadModel(); + // 创建池子：每个线程独享 Predictor + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + int predictorPoolSize = config.getPredictorPoolSize(); + if(config.getPredictorPoolSize() <= 0){ + predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数 + } + predictorPool.setMaxTotal(predictorPoolSize); + log.debug("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前引擎: " + Engine.getInstance().getEngineName()); + log.debug("模型推理器线程池最大数量: " + predictorPoolSize); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new DetectionException("模型加载失败", e); + } + } + + + /** + * 模型核心推理方法 + * @param image + * @return + */ + @Override + public R detect(Image image) { + Predictor predictor = null; + try { + predictor = predictorPool.borrowObject(); + Joints[] joints = predictor.predict(image); + return R.ok(joints); + } catch (Exception e) { + throw new DetectionException("行人检测错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + log.debug("释放资源"); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + } + + @Override + public Image detectAndDraw(Image image) { + Image drawnImage = ImageUtils.copy(image); + R allJoints = detect(drawnImage); + for (Joints joints : allJoints.getData()) { + drawnImage.drawJoints(joints); + } + return drawnImage; + } + + @Override + public R detectAndDraw(String imagePath, String outputPath) { + try { + Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); + R allJoints = detect(img); + for (Joints joints : allJoints.getData()) { + img.drawJoints(joints); + } + // 调用 save 方法将 Image 写入字节流 + img.save(Files.newOutputStream(Paths.get(outputPath)), "png"); + return allJoints; + } catch (IOException e) { + throw new ObbDetException(e); + } + } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } +} diff --git a/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java new file mode 100644 index 0000000..a5e02a3 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java @@ -0,0 +1,112 @@ +package cn.smartjavaai.pose.model; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.obb.model.CommonObbDetModel; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.pose.config.PoseModelConfig; +import cn.smartjavaai.pose.enums.PoseModelEnum; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 姿态估计 模型工厂 + * @author dwj + */ +@Slf4j +public class PoseDetModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile PoseDetModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private PoseDetModelFactory() {} + + // 双重检查锁定的单例方法 + public static PoseDetModelFactory getInstance() { + if (instance == null) { + synchronized (PoseDetModelFactory.class) { + if (instance == null) { + instance = new PoseDetModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public PoseModel getModel(PoseModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private PoseModel createFaceDetModel(PoseModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + PoseModel model = null; + try { + model = (PoseModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(PoseModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(PoseModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(PoseModelEnum.YOLOV8N_POSE_ONNX, CommonPoseModel.class); + registerAlgorithm(PoseModelEnum.YOLO11N_POSE_ONNX, CommonPoseModel.class); + registerAlgorithm(PoseModelEnum.YOLOV8N_POSE_PT, CommonPoseModel.class); + registerAlgorithm(PoseModelEnum.YOLO11N_POSE_PT, CommonPoseModel.class); +// registerAlgorithm(PoseModelEnum.SIMPLE_POSE_MXNET, CommonPoseModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/pose/model/PoseModel.java b/vision/src/main/java/cn/smartjavaai/pose/model/PoseModel.java new file mode 100644 index 0000000..0017216 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/pose/model/PoseModel.java @@ -0,0 +1,53 @@ +package cn.smartjavaai.pose.model; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Joints; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.objectdetection.config.PersonDetModelConfig; +import cn.smartjavaai.pose.config.PoseModelConfig; + +/** + * 姿态估计模型 + * @author dwj + */ +public interface PoseModel extends AutoCloseable{ + + + /** + * 加载模型 + * @param config + */ + void loadModel(PoseModelConfig config); + + /** + * 姿态估计 + * @param image + * @return + */ + default R detect(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 姿态估计并绘制 + * @param image + * @return + */ + default Image detectAndDraw(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 姿态估计并绘制 + * @param imagePath + * @param outputPath + * @return + */ + default R detectAndDraw(String imagePath, String outputPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/semseg/model/CommonSemSegModel.java b/vision/src/main/java/cn/smartjavaai/semseg/model/CommonSemSegModel.java index a392f54..5db097d 100644 --- a/vision/src/main/java/cn/smartjavaai/semseg/model/CommonSemSegModel.java +++ b/vision/src/main/java/cn/smartjavaai/semseg/model/CommonSemSegModel.java @@ -6,23 +6,32 @@ import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; import ai.djl.modality.cv.output.CategoryMask; +import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.pool.PredictorFactory; import cn.smartjavaai.common.utils.Base64ImageUtils; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.instanceseg.exception.InstanceSegException; import cn.smartjavaai.objectdetection.exception.DetectionException; import cn.smartjavaai.semseg.config.SemSegModelConfig; import cn.smartjavaai.semseg.criteria.SemSegCriteriaFactory; import cn.smartjavaai.vision.utils.CategoryMaskFilter; +import cn.smartjavaai.vision.utils.DetectorUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPool; +import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Objects; /** @@ -63,26 +72,12 @@ public class CommonSemSegModel implements SemSegModel { } } - @Override - public R detectBase64(String base64Image) { - if(StringUtils.isBlank(base64Image)){ - return R.fail(R.Status.INVALID_IMAGE); - } - try { - byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); - Image image = ImageFactory.getInstance().fromInputStream(new ByteArrayInputStream(imageData)); - return detect(image); - } catch (IOException e) { - throw new DetectionException("读取图片异常", e); - } - } - @Override public R detect(Image image) { CategoryMask categoryMask = detectCore(image); // 过滤 if(CollectionUtils.isNotEmpty(config.getAllowedClasses()) - && Objects.nonNull(categoryMask) && !categoryMask.getClasses().isEmpty()){ + && Objects.nonNull(categoryMask) && CollectionUtils.isNotEmpty(categoryMask.getClasses())){ categoryMask = new CategoryMaskFilter(config.getAllowedClasses()).filter(categoryMask); } return R.ok(categoryMask); @@ -117,6 +112,33 @@ public class CommonSemSegModel implements SemSegModel { } } + @Override + public R detectAndDraw(String imagePath, String outputPath) { + try { + Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); + CategoryMask categoryMask = detectCore(img); + if(Objects.isNull(categoryMask) || CollectionUtils.isEmpty(categoryMask.getClasses())){ + throw new InstanceSegException("未检测到实例"); + } + ImageUtils.drawMask(categoryMask, img, 180, 0); + img.save(Files.newOutputStream(Paths.get(outputPath)), "png"); + return R.ok(categoryMask); + } catch (IOException e) { + throw new InstanceSegException(e); + } + } + + @Override + public Image detectAndDraw(Image image) { + CategoryMask categoryMask = detectCore(image); + if(Objects.isNull(categoryMask) || CollectionUtils.isEmpty(categoryMask.getClasses())){ + throw new InstanceSegException("未检测到实例"); + } + Image drawnImage = ImageUtils.copy(image); + ImageUtils.drawMask(categoryMask, drawnImage, 180, 0); + return drawnImage; + } + @Override public void close() throws Exception { try { diff --git a/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModel.java b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModel.java index dfac361..37445a8 100644 --- a/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModel.java +++ b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModel.java @@ -20,15 +20,6 @@ public interface SemSegModel extends AutoCloseable{ */ void loadModel(SemSegModelConfig config); - /** - * 语义分割 - * @param base64Image - * @return - */ - default R detectBase64(String base64Image){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - /** * 语义分割 * @param image @@ -38,5 +29,14 @@ public interface SemSegModel extends AutoCloseable{ throw new UnsupportedOperationException("默认不支持该功能"); } + default R detectAndDraw(String imagePath, String outputPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default Image detectAndDraw(Image image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + } diff --git a/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java new file mode 100644 index 0000000..0099541 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java @@ -0,0 +1,107 @@ +package cn.smartjavaai.semseg.model; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.semseg.config.SemSegModelConfig; +import cn.smartjavaai.semseg.enums.SemSegModelEnum; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 语义分割 模型工厂 + * @author dwj + */ +@Slf4j +public class SemSegModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile SemSegModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private SemSegModelFactory() {} + + // 双重检查锁定的单例方法 + public static SemSegModelFactory getInstance() { + if (instance == null) { + synchronized (SemSegModelFactory.class) { + if (instance == null) { + instance = new SemSegModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public SemSegModel getModel(SemSegModelConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceDetModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private SemSegModel createFaceDetModel(SemSegModelConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + SemSegModel model = null; + try { + model = (SemSegModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(SemSegModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(SemSegModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(SemSegModelEnum.DEEPLABV3, CommonSemSegModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} + diff --git a/vision/src/main/java/cn/smartjavaai/vision/utils/DetectorUtils.java b/vision/src/main/java/cn/smartjavaai/vision/utils/DetectorUtils.java index 447b786..aeb8904 100644 --- a/vision/src/main/java/cn/smartjavaai/vision/utils/DetectorUtils.java +++ b/vision/src/main/java/cn/smartjavaai/vision/utils/DetectorUtils.java @@ -6,6 +6,7 @@ import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.modality.cv.output.Mask; import ai.djl.modality.cv.output.Rectangle; import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.Point; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.obb.entity.ObbResult; import cn.smartjavaai.obb.entity.YoloRotatedBox; @@ -15,6 +16,7 @@ import org.opencv.core.Scalar; import org.opencv.imgproc.Imgproc; import javax.imageio.ImageIO; +import java.awt.*; import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; @@ -94,7 +96,7 @@ public class DetectorUtils { /** - * 绘制文本框及文本 + * 绘制旋转框及文本（opencv） * @param srcMat * @param rotatedBoxeList */ @@ -111,4 +113,60 @@ public class DetectorUtils { } + /** + * 绘制旋转框及文本（BufferedImage） + * @param image + * @param rotatedBoxeList + */ + public static void drawRectWithText(BufferedImage image, List rotatedBoxeList) { + Graphics2D g2d = image.createGraphics(); + + // 抗锯齿，让线条和文字更平滑 + g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + g2d.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON); + + // 在左上角画文字，带背景 + FontMetrics fm = g2d.getFontMetrics(); + + for (YoloRotatedBox box : rotatedBoxeList) { + // 画笔颜色和字体 + g2d.setColor(Color.GREEN); + g2d.setStroke(new BasicStroke(1.5f)); // 线条粗细 + g2d.setFont(new Font("SansSerif", Font.PLAIN, 16)); // 设置中文字体，避免乱码 + List points = box.toPoints(); + // 画矩形的4条边 + g2d.drawLine((int) points.get(0).getX(), (int) points.get(0).getY(), (int) points.get(1).getX(), (int) points.get(1).getY()); + g2d.drawLine((int) points.get(1).getX(), (int) points.get(1).getY(), (int) points.get(2).getX(), (int) points.get(2).getY()); + g2d.drawLine((int) points.get(2).getX(), (int) points.get(2).getY(), (int) points.get(3).getX(), (int) points.get(3).getY()); + g2d.drawLine((int) points.get(3).getX(), (int) points.get(3).getY(), (int) points.get(0).getX(), (int) points.get(0).getY()); + int percent = (int) Math.round(box.score * 100); + String className = box.className + " " + percent + "%"; + int textWidth = fm.stringWidth(box.className + " " + box.score); + int textHeight = fm.getHeight(); + // 文字位置 + int textX = (int) points.get(0).getX(); + int textY = (int) points.get(0).getY() - 5; + // 画背景矩形（半透明） + g2d.setColor(new Color(0, 0, 0, 128)); + g2d.fillRect(textX, textY - textHeight, textWidth, textHeight); + g2d.setColor(Color.WHITE); + // 在左上角画文字 + g2d.drawString(className, (int) points.get(0).getX(), (int) points.get(0).getY() - 5); + } + g2d.dispose(); + } + + /** + * 绘制旋转框及文本 + * @param image + * @param rotatedBoxeList + */ + public static void drawRectWithText(Image image, List rotatedBoxeList){ + if(image.getWrappedImage() instanceof BufferedImage){ + drawRectWithText((BufferedImage) image.getWrappedImage(), rotatedBoxeList); + }else{ + drawRectWithText((Mat)image.getWrappedImage(), rotatedBoxeList); + } + } + } diff --git a/vision/src/main/java/cn/smartjavaai/vision/utils/ObbResultFilter.java b/vision/src/main/java/cn/smartjavaai/vision/utils/ObbResultFilter.java index 298d139..694420b 100644 --- a/vision/src/main/java/cn/smartjavaai/vision/utils/ObbResultFilter.java +++ b/vision/src/main/java/cn/smartjavaai/vision/utils/ObbResultFilter.java @@ -2,6 +2,7 @@ package cn.smartjavaai.vision.utils; import cn.smartjavaai.obb.entity.ObbResult; import cn.smartjavaai.obb.entity.YoloRotatedBox; +import org.apache.commons.collections.CollectionUtils; import java.util.Collections; import java.util.List; @@ -26,11 +27,12 @@ public class ObbResultFilter { return new ObbResult(Collections.emptyList()); } - // 1. 按类别过滤 - List filtered = result.getRotatedBoxeList().stream() - .filter(box -> allowedClasses.contains(box.className)) - .collect(Collectors.toList()); - + List filtered = result.getRotatedBoxeList(); + if(CollectionUtils.isNotEmpty(allowedClasses)){ + filtered = result.getRotatedBoxeList().stream() + .filter(box -> allowedClasses.contains(box.className)) + .collect(Collectors.toList()); + } // 2. 按 score 排序 // filtered.sort((a, b) -> Float.compare(b.score, a.score));