mirror of
https://github.com/geekwenjie/SmartJavaAI.git
synced 2026-09-20 01:29:18 +00:00
【通用视觉】集成 OpenAI CLIP 模型，支持以图搜图、以文搜图、以图搜文等功能
【通用视觉】新增 YOLO 图像分类模型支持 【ASR/TTS】集成 Sherpa TTS（语音合成）与 ASR（语音识别）模块，支持中文、粤语、方言、英文等多种语言 【目标检测】优化视频目标检测功能
This commit is contained in:
dengwenjie
@@ -0,0 +1,45 @@
|
||||
package cn.smartjavaai.clip.config;
|
||||
|
||||
import cn.smartjavaai.action.enums.ActionRecModelEnum;
|
||||
import cn.smartjavaai.clip.enums.ClipModelEnum;
|
||||
import cn.smartjavaai.common.config.ModelConfig;
|
||||
import cn.smartjavaai.common.enums.DeviceEnum;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* CLIP模型参数配置
|
||||
*
|
||||
* @author dwj
|
||||
*/
|
||||
@Data
|
||||
public class ClipModelConfig extends ModelConfig {
|
||||
|
||||
/**
|
||||
* 模型
|
||||
*/
|
||||
private ClipModelEnum modelEnum;
|
||||
|
||||
|
||||
/**
|
||||
* 模型路径
|
||||
*/
|
||||
private String modelPath;
|
||||
|
||||
|
||||
|
||||
public ClipModelConfig() {
|
||||
}
|
||||
|
||||
public ClipModelConfig(ClipModelEnum modelEnum, DeviceEnum device) {
|
||||
this.modelEnum = modelEnum;
|
||||
setDevice(device);
|
||||
}
|
||||
|
||||
public ClipModelConfig(ClipModelEnum modelEnum) {
|
||||
this.modelEnum = modelEnum;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package cn.smartjavaai.clip.enums;
|
||||
|
||||
/**
|
||||
* CLIP模型枚举
|
||||
* @author dwj
|
||||
*/
|
||||
public enum ClipModelEnum {
|
||||
|
||||
OPENAI;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 根据名称获取枚举 (忽略大小写和下划线变体)
|
||||
*/
|
||||
public static ClipModelEnum fromName(String name) {
|
||||
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
|
||||
for (ClipModelEnum model : values()) {
|
||||
if (model.name().replaceAll("_", "").equals(formatted)) {
|
||||
return model;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("未知模型名称: " + name);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package cn.smartjavaai.clip.exception;
|
||||
|
||||
/**
|
||||
* CLIP异常
|
||||
* @author dwj
|
||||
*/
|
||||
public class ClipException extends RuntimeException{
|
||||
|
||||
public ClipException() {
|
||||
super();
|
||||
}
|
||||
|
||||
public ClipException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
}
|
||||
|
||||
public ClipException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public ClipException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ClipException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
}
|
||||
133
vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java
Normal file
133
vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java
Normal file
@@ -0,0 +1,133 @@
|
||||
package cn.smartjavaai.clip.model;
|
||||
|
||||
import ai.djl.modality.cv.Image;
|
||||
import cn.smartjavaai.clip.config.ClipModelConfig;
|
||||
import cn.smartjavaai.common.entity.R;
|
||||
|
||||
/**
|
||||
* @author dwj
|
||||
* @date 2025/10/20
|
||||
*/
|
||||
public interface ClipModel extends AutoCloseable{
|
||||
|
||||
/**
|
||||
* 加载模型
|
||||
* @param config
|
||||
*/
|
||||
void loadModel(ClipModelConfig config); // 加载模型
|
||||
|
||||
/**
|
||||
* 图片特征提取
|
||||
* @param image
|
||||
* @return
|
||||
*/
|
||||
default R<float[]> extractImageFeatures(Image image){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片特征提取
|
||||
* @param imagePath
|
||||
* @return
|
||||
*/
|
||||
default R<float[]> extractImageFeatures(String imagePath){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文本特征提取
|
||||
* @param inputs
|
||||
* @return
|
||||
*/
|
||||
default R<float[]> extractTextFeatures(String inputs){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文本和图片特征比较
|
||||
* @param image
|
||||
* @param text
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareTextAndImage(Image image, String text){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片特征比较
|
||||
* @param image1 图1
|
||||
* @param image2 图2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareImage(Image image1, Image image2){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 图片特征比较
|
||||
* @param image1 图1
|
||||
* @param image2 图2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareImage(Image image1, Image image2, float scale){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片特征比较
|
||||
* @param imagePath1 图1
|
||||
* @param imagePath2 图2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareImage(String imagePath1, String imagePath2){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片特征比较
|
||||
* @param imagePath1 图1
|
||||
* @param imagePath2 图2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareImage(String imagePath1, String imagePath2, float scale){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 文本特征比较
|
||||
* @param input1 文本1
|
||||
* @param input2 文本2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareText(String input1, String input2){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文本特征比较
|
||||
* @param input1 文本1
|
||||
* @param input2 文本2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareText(String input1, String input2, float scale){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
/**
|
||||
* 文本特征比较
|
||||
* @param feature1 文本1
|
||||
* @param feature2 文本2
|
||||
* @return
|
||||
*/
|
||||
default R<Float> compareFeatures(float[] feature1, float[] feature2, float scale){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
default void setFromFactory(boolean fromFactory){
|
||||
throw new UnsupportedOperationException("默认不支持该功能");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package cn.smartjavaai.clip.model;
|
||||
|
||||
import cn.smartjavaai.action.model.ActionRecModelFactory;
|
||||
import cn.smartjavaai.action.model.CommonActionRecModel;
|
||||
import cn.smartjavaai.clip.config.ClipModelConfig;
|
||||
import cn.smartjavaai.clip.enums.ClipModelEnum;
|
||||
import cn.smartjavaai.common.config.Config;
|
||||
import cn.smartjavaai.objectdetection.exception.DetectionException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* @author dwj
|
||||
* @date 2025/10/20
|
||||
*/
|
||||
@Slf4j
|
||||
public class ClipModelFactory {
|
||||
|
||||
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
|
||||
private static volatile ClipModelFactory instance;
|
||||
|
||||
private static final ConcurrentHashMap<ClipModelEnum, ClipModel> modelMap = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 模型注册表
|
||||
*/
|
||||
private static final Map<ClipModelEnum, Class<? extends ClipModel>> registry =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
// 私有构造函数，防止外部创建实例
|
||||
private ClipModelFactory() {}
|
||||
|
||||
// 双重检查锁定的单例方法
|
||||
public static ClipModelFactory getInstance() {
|
||||
if (instance == null) {
|
||||
synchronized (ClipModelFactory.class) {
|
||||
if (instance == null) {
|
||||
instance = new ClipModelFactory();
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取模型（通过配置）
|
||||
* @param config
|
||||
* @return
|
||||
*/
|
||||
public ClipModel getModel(ClipModelConfig config) {
|
||||
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
|
||||
throw new DetectionException("未配置模型");
|
||||
}
|
||||
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
|
||||
return createFaceDetModel(config);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用ModelConfig创建模型
|
||||
* @param config
|
||||
* @return
|
||||
*/
|
||||
private ClipModel createFaceDetModel(ClipModelConfig config) {
|
||||
Class<?> clazz = registry.get(config.getModelEnum());
|
||||
if(clazz == null){
|
||||
throw new DetectionException("Unsupported model");
|
||||
}
|
||||
ClipModel model = null;
|
||||
try {
|
||||
model = (ClipModel) clazz.newInstance();
|
||||
} catch (InstantiationException | IllegalAccessException e) {
|
||||
throw new DetectionException(e);
|
||||
}
|
||||
model.loadModel(config);
|
||||
model.setFromFactory(true);
|
||||
return model;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 注册模型
|
||||
* @param modelEnum
|
||||
* @param clazz
|
||||
*/
|
||||
private static void registerAlgorithm(ClipModelEnum modelEnum, Class<? extends ClipModel> clazz) {
|
||||
registry.put(modelEnum, clazz);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 初始化默认算法
|
||||
static {
|
||||
registerAlgorithm(ClipModelEnum.OPENAI, OpenAIClipModel.class);
|
||||
log.debug("缓存目录：{}", Config.getCachePath());
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭所有已加载的模型
|
||||
*/
|
||||
public void closeAll() {
|
||||
modelMap.values().forEach(model -> {
|
||||
try {
|
||||
model.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
});
|
||||
modelMap.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除缓存的模型
|
||||
* @param modelEnum
|
||||
*/
|
||||
public static void removeFromCache(ClipModelEnum modelEnum) {
|
||||
modelMap.remove(modelEnum);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package cn.smartjavaai.clip.model;
|
||||
|
||||
import ai.djl.Device;
|
||||
import ai.djl.MalformedModelException;
|
||||
import ai.djl.engine.Engine;
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
|
||||
import ai.djl.inference.Predictor;
|
||||
import ai.djl.modality.cv.Image;
|
||||
import ai.djl.ndarray.NDList;
|
||||
import ai.djl.repository.zoo.Criteria;
|
||||
import ai.djl.repository.zoo.ModelNotFoundException;
|
||||
import ai.djl.repository.zoo.ZooModel;
|
||||
import ai.djl.translate.NoopTranslator;
|
||||
import ai.djl.util.Pair;
|
||||
import cn.smartjavaai.clip.config.ClipModelConfig;
|
||||
import cn.smartjavaai.clip.exception.ClipException;
|
||||
import cn.smartjavaai.clip.pool.ClipImagePredictorFactory;
|
||||
import cn.smartjavaai.clip.pool.ClipImageTextPredictorFactory;
|
||||
import cn.smartjavaai.clip.pool.ClipTextPredictorFactory;
|
||||
import cn.smartjavaai.clip.translator.ImageTextTranslator;
|
||||
import cn.smartjavaai.clip.translator.ImageTranslator;
|
||||
import cn.smartjavaai.clip.translator.TextTranslator;
|
||||
import cn.smartjavaai.common.cv.SmartImageFactory;
|
||||
import cn.smartjavaai.common.entity.R;
|
||||
import cn.smartjavaai.common.enums.DeviceEnum;
|
||||
import cn.smartjavaai.common.enums.SimilarityType;
|
||||
import cn.smartjavaai.common.pool.PredictorFactory;
|
||||
import cn.smartjavaai.common.utils.DJLCommonUtils;
|
||||
import cn.smartjavaai.common.utils.ImageUtils;
|
||||
import cn.smartjavaai.common.utils.SimilarityUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.pool2.impl.GenericObjectPool;
|
||||
|
||||
import javax.sound.sampled.Clip;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* openai clip 模型
|
||||
* @author dwj
|
||||
*/
|
||||
@Slf4j
|
||||
public class OpenAIClipModel implements ClipModel{
|
||||
|
||||
private ClipModelConfig config;
|
||||
|
||||
private ZooModel<NDList, NDList> model;
|
||||
|
||||
private HuggingFaceTokenizer tokenizer;
|
||||
|
||||
private GenericObjectPool<Predictor<Image, float[]>> imageFeaturePredictorPool;
|
||||
|
||||
private GenericObjectPool<Predictor<String, float[]>> textFeaturePredictorPool;
|
||||
|
||||
private GenericObjectPool<Predictor<Pair<Image, String>, float[]>> imgTextPredictorPool;
|
||||
|
||||
@Override
|
||||
public void loadModel(ClipModelConfig config) {
|
||||
if(Objects.isNull(config)){
|
||||
throw new ClipException("config为null");
|
||||
}
|
||||
if(StringUtils.isBlank(config.getModelPath())){
|
||||
throw new ClipException("modelPath为空");
|
||||
}
|
||||
this.config = config;
|
||||
try {
|
||||
// Device device = null;
|
||||
// if(!Objects.isNull(config.getDevice())){
|
||||
// device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
|
||||
// }
|
||||
boolean isUrl = DJLCommonUtils.hasSupportedProtocol(config.getModelPath());
|
||||
Criteria<NDList, NDList> criteria =
|
||||
Criteria.builder()
|
||||
.setTypes(NDList.class, NDList.class)
|
||||
// .optModelUrls("https://resources.djl.ai/demo/pytorch/clip.zip")
|
||||
.optModelUrls(isUrl ? config.getModelPath() : null)
|
||||
.optModelName("clip.pt")
|
||||
.optModelPath(isUrl ? null : Paths.get(config.getModelPath()))
|
||||
.optTranslator(new NoopTranslator())
|
||||
.optEngine("PyTorch")
|
||||
// .optOption("mapLocation", "true")
|
||||
.optDevice(Device.cpu()) // torchscript model only support CPU
|
||||
.build();
|
||||
model = criteria.loadModel();
|
||||
Path modelCachePath = model.getWrappedModel().getModelPath();
|
||||
Path tokenizerPath = modelCachePath.resolve("tokenizer.json");
|
||||
tokenizer = HuggingFaceTokenizer.newInstance(tokenizerPath);
|
||||
// 创建池子：每个线程独享 Predictor
|
||||
imageFeaturePredictorPool = new GenericObjectPool<>(new ClipImagePredictorFactory(model));
|
||||
textFeaturePredictorPool = new GenericObjectPool<>(new ClipTextPredictorFactory(model, tokenizer));
|
||||
imgTextPredictorPool = new GenericObjectPool<>(new ClipImageTextPredictorFactory(model, tokenizer));
|
||||
int predictorPoolSize = config.getPredictorPoolSize();
|
||||
if(config.getPredictorPoolSize() <= 0){
|
||||
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
|
||||
}
|
||||
imageFeaturePredictorPool.setMaxTotal(predictorPoolSize);
|
||||
textFeaturePredictorPool.setMaxTotal(predictorPoolSize);
|
||||
imgTextPredictorPool.setMaxTotal(predictorPoolSize);
|
||||
log.debug("当前设备: " + model.getNDManager().getDevice());
|
||||
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
|
||||
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
|
||||
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
|
||||
throw new ClipException("模型加载失败", e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<float[]> extractImageFeatures(Image image) {
|
||||
Predictor<Image, float[]> predictor = null;
|
||||
try {
|
||||
predictor = imageFeaturePredictorPool.borrowObject();
|
||||
return R.ok(predictor.predict(image));
|
||||
} catch (Exception e) {
|
||||
throw new ClipException("特征提取错误", e);
|
||||
}finally {
|
||||
if (predictor != null) {
|
||||
try {
|
||||
imageFeaturePredictorPool.returnObject(predictor); //归还
|
||||
} catch (Exception e) {
|
||||
log.warn("归还Predictor失败", e);
|
||||
try {
|
||||
predictor.close(); // 归还失败才销毁
|
||||
} catch (Exception ex) {
|
||||
log.error("关闭Predictor失败", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<float[]> extractImageFeatures(String imagePath) {
|
||||
Image image = null;
|
||||
try {
|
||||
image = SmartImageFactory.getInstance().fromFile(imagePath);
|
||||
return extractImageFeatures(image);
|
||||
} catch (IOException e) {
|
||||
throw new ClipException(e);
|
||||
} finally {
|
||||
ImageUtils.releaseOpenCVMat(image);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<float[]> extractTextFeatures(String inputs) {
|
||||
Predictor<String, float[]> predictor = null;
|
||||
try {
|
||||
predictor = textFeaturePredictorPool.borrowObject();
|
||||
return R.ok(predictor.predict(inputs));
|
||||
} catch (Exception e) {
|
||||
throw new ClipException("特征提取错误", e);
|
||||
}finally {
|
||||
if (predictor != null) {
|
||||
try {
|
||||
textFeaturePredictorPool.returnObject(predictor); //归还
|
||||
} catch (Exception e) {
|
||||
log.warn("归还Predictor失败", e);
|
||||
try {
|
||||
predictor.close(); // 归还失败才销毁
|
||||
} catch (Exception ex) {
|
||||
log.error("关闭Predictor失败", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareTextAndImage(Image image, String text) {
|
||||
Predictor<Pair<Image, String>, float[]> predictor = null;
|
||||
try {
|
||||
predictor = imgTextPredictorPool.borrowObject();
|
||||
float[] imageFeatures = predictor.predict(new Pair<>(image, text));
|
||||
if (imageFeatures == null || imageFeatures.length == 0){
|
||||
return R.fail(R.Status.Unknown.getCode(), "特征为空");
|
||||
}
|
||||
return R.ok(imageFeatures[0]);
|
||||
} catch (Exception e) {
|
||||
throw new ClipException("特征提取错误", e);
|
||||
}finally {
|
||||
if (predictor != null) {
|
||||
try {
|
||||
imgTextPredictorPool.returnObject(predictor); //归还
|
||||
} catch (Exception e) {
|
||||
log.warn("归还Predictor失败", e);
|
||||
try {
|
||||
predictor.close(); // 归还失败才销毁
|
||||
} catch (Exception ex) {
|
||||
log.error("关闭Predictor失败", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareFeatures(float[] feature1, float[] feature2, float scale) {
|
||||
float similarity = SimilarityUtil.calculate(feature1, feature2, SimilarityType.COSINE, false);
|
||||
return R.ok(similarity * scale);
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareImage(Image image1, Image image2) {
|
||||
return compareImage(image1, image2, 1.0f);
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareImage(Image image1, Image image2, float scale) {
|
||||
R<float[]> features1 = extractImageFeatures(image1);
|
||||
R<float[]> features2 = extractImageFeatures(image2);
|
||||
if(!features1.isSuccess()){
|
||||
return R.fail(features1.getCode(), features1.getMessage());
|
||||
}
|
||||
if(!features2.isSuccess()){
|
||||
return R.fail(features2.getCode(), features2.getMessage());
|
||||
}
|
||||
return compareFeatures(features1.getData(), features2.getData(), scale);
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareImage(String imagePath1, String imagePath2) {
|
||||
return compareImage(imagePath1, imagePath2, 1.0f);
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareImage(String imagePath1, String imagePath2, float scale) {
|
||||
Image image1 = null;
|
||||
Image image2 = null;
|
||||
try {
|
||||
image1 = SmartImageFactory.getInstance().fromFile(imagePath1);
|
||||
image2 = SmartImageFactory.getInstance().fromFile(imagePath2);
|
||||
return compareImage(image1, image2, scale);
|
||||
} catch (IOException e) {
|
||||
throw new ClipException(e);
|
||||
} finally {
|
||||
ImageUtils.releaseOpenCVMat(image1);
|
||||
ImageUtils.releaseOpenCVMat(image2);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareText(String input1, String input2) {
|
||||
return compareText(input1, input2, 1.0f);
|
||||
}
|
||||
|
||||
@Override
|
||||
public R<Float> compareText(String input1, String input2, float scale) {
|
||||
R<float[]> features1 = extractTextFeatures(input1);
|
||||
R<float[]> features2 = extractTextFeatures(input2);
|
||||
if(!features1.isSuccess()){
|
||||
return R.fail(features1.getCode(), features1.getMessage());
|
||||
}
|
||||
if(!features2.isSuccess()){
|
||||
return R.fail(features2.getCode(), features2.getMessage());
|
||||
}
|
||||
return compareFeatures(features1.getData(), features2.getData(), scale);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
try {
|
||||
if (imageFeaturePredictorPool != null) {
|
||||
imageFeaturePredictorPool.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("关闭 predictorPool 失败", e);
|
||||
}
|
||||
try {
|
||||
if (textFeaturePredictorPool != null) {
|
||||
textFeaturePredictorPool.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("关闭 predictorPool 失败", e);
|
||||
}
|
||||
try {
|
||||
if (imgTextPredictorPool != null) {
|
||||
imgTextPredictorPool.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("关闭 predictorPool 失败", e);
|
||||
}
|
||||
try {
|
||||
if (model != null) {
|
||||
model.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("关闭 model 失败", e);
|
||||
}
|
||||
try {
|
||||
if (tokenizer != null) {
|
||||
tokenizer.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("关闭 tokenizer 失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean fromFactory = false;
|
||||
public boolean isFromFactory() {
|
||||
return fromFactory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFromFactory(boolean fromFactory) {
|
||||
this.fromFactory = fromFactory;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package cn.smartjavaai.clip.pool;
|
||||
|
||||
import ai.djl.Model;
|
||||
import ai.djl.inference.Predictor;
|
||||
import ai.djl.modality.cv.Image;
|
||||
import cn.smartjavaai.clip.translator.ImageTranslator;
|
||||
import org.apache.commons.pool2.BasePooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
/**
|
||||
* @author dwj
|
||||
* @date 2025/10/20
|
||||
*/
|
||||
public class ClipImagePredictorFactory extends BasePooledObjectFactory<Predictor<Image, float[]>> {
|
||||
private final Model model;
|
||||
|
||||
public ClipImagePredictorFactory(Model model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Predictor<Image, float[]> create() {
|
||||
return model.newPredictor(new ImageTranslator());
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<Predictor<Image, float[]>> wrap(Predictor<Image, float[]> predictor) {
|
||||
return new DefaultPooledObject<>(predictor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(PooledObject<Predictor<Image, float[]>> p) {
|
||||
p.getObject().close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package cn.smartjavaai.clip.pool;
|
||||
|
||||
import ai.djl.Model;
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
|
||||
import ai.djl.inference.Predictor;
|
||||
import ai.djl.modality.cv.Image;
|
||||
import ai.djl.util.Pair;
|
||||
import cn.smartjavaai.clip.translator.ImageTextTranslator;
|
||||
import org.apache.commons.pool2.BasePooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
/**
|
||||
* @author dwj
|
||||
*/
|
||||
public class ClipImageTextPredictorFactory extends BasePooledObjectFactory<Predictor<Pair<Image, String>, float[]>> {
|
||||
|
||||
private final Model model;
|
||||
|
||||
private final HuggingFaceTokenizer tokenizer;
|
||||
|
||||
public ClipImageTextPredictorFactory(Model model, HuggingFaceTokenizer tokenizer) {
|
||||
this.model = model;
|
||||
this.tokenizer = tokenizer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Predictor<Pair<Image, String>, float[]> create() {
|
||||
return model.newPredictor(new ImageTextTranslator(tokenizer));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<Predictor<Pair<Image, String>, float[]>> wrap(Predictor<Pair<Image, String>, float[]> predictor) {
|
||||
return new DefaultPooledObject<>(predictor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(PooledObject<Predictor<Pair<Image, String>, float[]>> p) {
|
||||
p.getObject().close();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package cn.smartjavaai.clip.pool;
|
||||
|
||||
import ai.djl.Model;
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
|
||||
import ai.djl.inference.Predictor;
|
||||
import ai.djl.modality.cv.Image;
|
||||
import ai.djl.modality.nlp.preprocess.Tokenizer;
|
||||
import cn.smartjavaai.clip.translator.ImageTranslator;
|
||||
import cn.smartjavaai.clip.translator.TextTranslator;
|
||||
import org.apache.commons.pool2.BasePooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
/**
|
||||
* @author dwj
|
||||
*/
|
||||
public class ClipTextPredictorFactory extends BasePooledObjectFactory<Predictor<String, float[]>> {
|
||||
private final Model model;
|
||||
private final HuggingFaceTokenizer tokenizer;
|
||||
|
||||
public ClipTextPredictorFactory(Model model, HuggingFaceTokenizer tokenizer) {
|
||||
this.model = model;
|
||||
this.tokenizer = tokenizer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Predictor<String, float[]> create() {
|
||||
return model.newPredictor(new TextTranslator(tokenizer));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<Predictor<String, float[]>> wrap(Predictor<String, float[]> predictor) {
|
||||
return new DefaultPooledObject<>(predictor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(PooledObject<Predictor<String, float[]>> p) {
|
||||
p.getObject().close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
|
||||
* with the License. A copy of the License is located at
|
||||
*
|
||||
* http://aws.amazon.com/apache2.0/
|
||||
*
|
||||
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*/
|
||||
package cn.smartjavaai.clip.translator;
|
||||
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
|
||||
import ai.djl.modality.cv.Image;
|
||||
import ai.djl.ndarray.NDArray;
|
||||
import ai.djl.ndarray.NDList;
|
||||
import ai.djl.translate.NoBatchifyTranslator;
|
||||
import ai.djl.translate.TranslatorContext;
|
||||
import ai.djl.util.Pair;
|
||||
|
||||
public class ImageTextTranslator implements NoBatchifyTranslator<Pair<Image, String>, float[]> {
|
||||
|
||||
private ImageTranslator imgTranslator;
|
||||
private TextTranslator txtTranslator;
|
||||
|
||||
HuggingFaceTokenizer tokenizer;
|
||||
|
||||
public ImageTextTranslator(HuggingFaceTokenizer tokenizer) {
|
||||
this.imgTranslator = new ImageTranslator();
|
||||
this.txtTranslator = new TextTranslator(tokenizer);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public float[] processOutput(TranslatorContext ctx, NDList list) throws Exception {
|
||||
NDArray logitsPerImage = list.get(0);
|
||||
return logitsPerImage.toFloatArray();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public NDList processInput(TranslatorContext ctx, Pair<Image, String> input) throws Exception {
|
||||
NDList imageInput = imgTranslator.processInput(ctx, input.getKey());
|
||||
NDList textInput = txtTranslator.processInput(ctx, input.getValue());
|
||||
return new NDList(textInput.get(0), imageInput.get(0), textInput.get(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
|
||||
* with the License. A copy of the License is located at
|
||||
*
|
||||
* http://aws.amazon.com/apache2.0/
|
||||
*
|
||||
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*/
|
||||
package cn.smartjavaai.clip.translator;
|
||||
|
||||
import ai.djl.modality.cv.Image;
|
||||
import ai.djl.modality.cv.util.NDImageUtils;
|
||||
import ai.djl.ndarray.NDArray;
|
||||
import ai.djl.ndarray.NDList;
|
||||
import ai.djl.translate.NoBatchifyTranslator;
|
||||
import ai.djl.translate.TranslatorContext;
|
||||
|
||||
public class ImageTranslator implements NoBatchifyTranslator<Image, float[]> {
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public float[] processOutput(TranslatorContext ctx, NDList list) {
|
||||
NDArray array = list.singletonOrThrow();
|
||||
return array.toFloatArray();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public NDList processInput(TranslatorContext ctx, Image input) {
|
||||
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
|
||||
|
||||
float percent = 224f / Math.min(input.getWidth(), input.getHeight());
|
||||
int resizedWidth = Math.round(input.getWidth() * percent);
|
||||
int resizedHeight = Math.round(input.getHeight() * percent);
|
||||
|
||||
array =
|
||||
NDImageUtils.resize(
|
||||
array, resizedWidth, resizedHeight, Image.Interpolation.BICUBIC);
|
||||
array = NDImageUtils.centerCrop(array, 224, 224);
|
||||
array = NDImageUtils.toTensor(array);
|
||||
NDArray placeholder = ctx.getNDManager().create("");
|
||||
placeholder.setName("module_method:get_image_features");
|
||||
return new NDList(array.expandDims(0), placeholder);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
|
||||
* with the License. A copy of the License is located at
|
||||
*
|
||||
* http://aws.amazon.com/apache2.0/
|
||||
*
|
||||
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*/
|
||||
package cn.smartjavaai.clip.translator;
|
||||
|
||||
import ai.djl.huggingface.tokenizers.Encoding;
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
|
||||
import ai.djl.ndarray.NDArray;
|
||||
import ai.djl.ndarray.NDList;
|
||||
import ai.djl.translate.NoBatchifyTranslator;
|
||||
import ai.djl.translate.TranslatorContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
public class TextTranslator implements NoBatchifyTranslator<String, float[]> {
|
||||
|
||||
HuggingFaceTokenizer tokenizer;
|
||||
|
||||
public TextTranslator(HuggingFaceTokenizer tokenizer) {
|
||||
this.tokenizer = tokenizer;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public float[] processOutput(TranslatorContext ctx, NDList list) {
|
||||
return list.singletonOrThrow().toFloatArray();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public NDList processInput(TranslatorContext ctx, String input) {
|
||||
Encoding encoding = tokenizer.encode(input);
|
||||
NDArray attention = ctx.getNDManager().create(encoding.getAttentionMask());
|
||||
NDArray inputIds = ctx.getNDManager().create(encoding.getIds());
|
||||
NDArray placeholder = ctx.getNDManager().create("");
|
||||
placeholder.setName("module_method:get_text_features");
|
||||
return new NDList(inputIds.expandDims(0), attention.expandDims(0), placeholder);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user