- 新增OCR文字识别模块：支持最新 PP-OCRv5

- OCR文本识别：支持文字方向检测与自动校正

smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/utils/ImageUtils.java

@@ -1,221 +0,0 @@
-package cn.smartjavaai.ocr.utils;
-
-import ai.djl.modality.cv.Image;
-import ai.djl.modality.cv.ImageFactory;
-import ai.djl.modality.cv.output.DetectedObjects;
-import ai.djl.ndarray.NDArray;
-import cn.smartjavaai.common.entity.DetectionInfo;
-import cn.smartjavaai.common.entity.DetectionRectangle;
-import cn.smartjavaai.common.entity.DetectionResponse;
-import cn.smartjavaai.common.utils.OpenCVUtils;
-import org.opencv.core.Mat;
-import org.opencv.core.Point;
-import org.opencv.core.Scalar;
-import org.opencv.imgproc.Imgproc;
-
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * 图像工具类
- */
-public class ImageUtils {
-
-
-    /**
-     * 保存BufferedImage图片
-     *
-     * @param img
-     * @param name
-     * @param path
-     */
-    public static void saveImage(BufferedImage img, String name, String path) {
-        Image djlImg = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(img)); // 支持多种图片格式，自动适配
-        Path outputDir = Paths.get(path);
-        Path imagePath = outputDir.resolve(name);
-        // OpenJDK 不能保存 jpg 图片的 alpha channel
-        try {
-            djlImg.save(Files.newOutputStream(imagePath), "png");
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    /**
-     * 保存DJL图片
-     *
-     * @param img
-     * @param name
-     * @param path
-     */
-    public static void saveImage(Image img, String name, String path) {
-        Path outputDir = Paths.get(path);
-        Path imagePath = outputDir.resolve(name);
-        // OpenJDK 不能保存 jpg 图片的 alpha channel
-        try {
-            img.save(Files.newOutputStream(imagePath), "png");
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    /**
-     * 保存图片,含检测框
-     *
-     * @param img
-     * @param detection
-     * @param name
-     * @param path
-     * @throws IOException
-     */
-    public static void saveBoundingBoxImage(
-            Image img, DetectedObjects detection, String name, String path) throws IOException {
-        // Make image copy with alpha channel because original image was jpg
-        img.drawBoundingBoxes(detection);
-        Path outputDir = Paths.get(path);
-        Files.createDirectories(outputDir);
-        Path imagePath = outputDir.resolve(name);
-        // OpenJDK can't save jpg with alpha channel
-        img.save(Files.newOutputStream(imagePath), "png");
-    }
-
-
-
-
-    /**
-     * 画检测框(有倾斜角)
-     *
-     * @param image
-     * @param box
-     */
-    public static void drawImageRect(BufferedImage image, NDArray box) {
-        float[] points = box.toFloatArray();
-        int[] xPoints = new int[5];
-        int[] yPoints = new int[5];
-
-        for (int i = 0; i < 4; i++) {
-            xPoints[i] = (int) points[2 * i];
-            yPoints[i] = (int) points[2 * i + 1];
-        }
-        xPoints[4] = xPoints[0];
-        yPoints[4] = yPoints[0];
-
-        // 将绘制图像转换为Graphics2D
-        Graphics2D g = (Graphics2D) image.getGraphics();
-        try {
-            g.setColor(new Color(0, 255, 0));
-            // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角
-            BasicStroke bStroke = new BasicStroke(4, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER);
-            g.setStroke(bStroke);
-            g.drawPolyline(xPoints, yPoints, 5); // xPoints, yPoints, nPoints
-        } finally {
-            g.dispose();
-        }
-    }
-
-    /**
-     * 画检测框(有倾斜角)和文本
-     *
-     * @param image
-     * @param box
-     * @param text
-     */
-    public static void drawImageRectWithText(BufferedImage image, NDArray box, String text) {
-        float[] points = box.toFloatArray();
-        int[] xPoints = new int[5];
-        int[] yPoints = new int[5];
-
-        for (int i = 0; i < 4; i++) {
-            xPoints[i] = (int) points[2 * i];
-            yPoints[i] = (int) points[2 * i + 1];
-        }
-        xPoints[4] = xPoints[0];
-        yPoints[4] = yPoints[0];
-
-        // 将绘制图像转换为Graphics2D
-        Graphics2D g = (Graphics2D) image.getGraphics();
-        try {
-            int fontSize = 32;
-            Font font = new Font("楷体", Font.PLAIN, fontSize);
-            g.setFont(font);
-            g.setColor(new Color(0, 0, 255));
-            // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角
-            BasicStroke bStroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER);
-            g.setStroke(bStroke);
-            g.drawPolyline(xPoints, yPoints, 5); // xPoints, yPoints, nPoints
-            g.drawString(text, xPoints[0], yPoints[0]);
-        } finally {
-            g.dispose();
-        }
-    }
-
-    /**
-     * 画检测框
-     *
-     * @param image
-     * @param x
-     * @param y
-     * @param width
-     * @param height
-     */
-    public static void drawImageRect(BufferedImage image, int x, int y, int width, int height) {
-        // 将绘制图像转换为Graphics2D
-        Graphics2D g = (Graphics2D) image.getGraphics();
-        try {
-            g.setColor(new Color(0, 255, 0));
-            // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角
-            BasicStroke bStroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER);
-            g.setStroke(bStroke);
-            g.drawRect(x, y, width, height);
-        } finally {
-            g.dispose();
-        }
-    }
-
-    /**
-     * 显示文字
-     *
-     * @param image
-     * @param text
-     * @param x
-     * @param y
-     */
-    public static void drawImageText(BufferedImage image, String text, int x, int y) {
-        Graphics graphics = image.getGraphics();
-        int fontSize = 32;
-        Font font = new Font("楷体", Font.PLAIN, fontSize);
-        try {
-            graphics.setFont(font);
-            graphics.setColor(new Color(0, 0, 255));
-            int strWidth = graphics.getFontMetrics().stringWidth(text);
-            graphics.drawString(text, x, y);
-        } finally {
-            graphics.dispose();
-        }
-    }
-
-    /**
-     * 画矩形
-     *
-     * @param mat
-     * @param box
-     */
-    public static void drawRect(Mat mat, DetectionResponse detectionResponse) {
-
-        for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){
-            DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle();
-            // 左上角点
-            Point topLeft = new Point(detectionRectangle.getX(), detectionRectangle.getY());
-            // 右下角点
-            Point bottomRight = new Point(detectionRectangle.getX() + detectionRectangle.getWidth(), detectionRectangle.getY() + detectionRectangle.getHeight());
-            // 绘制矩形（图像、左上角、右下角、颜色（BGR）、线宽）
-            Imgproc.rectangle(mat, topLeft, bottomRight, new Scalar(0, 255, 0), 1);
-        }
-    }
-}

smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/utils/OcrUtils.java

@@ -1,15 +1,30 @@
 package cn.smartjavaai.ocr.utils;
 
 import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
 import ai.djl.modality.cv.output.BoundingBox;
 import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.modality.cv.util.NDImageUtils;
 import ai.djl.ndarray.NDArray;
 import ai.djl.ndarray.NDList;
-import cn.smartjavaai.common.entity.DetectionInfo;
-import cn.smartjavaai.common.entity.DetectionRectangle;
-import cn.smartjavaai.common.entity.DetectionResponse;
+import ai.djl.ndarray.NDManager;
+import ai.djl.opencv.OpenCVImageFactory;
+import cn.smartjavaai.common.entity.*;
+import cn.smartjavaai.common.entity.Point;
+import cn.smartjavaai.ocr.entity.OcrBox;
+import cn.smartjavaai.ocr.entity.OcrInfo;
+import cn.smartjavaai.ocr.entity.OcrItem;
+import cn.smartjavaai.ocr.entity.RotatedBoxCompX;
+import cn.smartjavaai.ocr.enums.AngleEnum;
+import cn.smartjavaai.ocr.opencv.OcrNDArrayUtils;
+import cn.smartjavaai.ocr.opencv.OcrOpenCVUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.opencv.core.Mat;
+import org.opencv.core.Scalar;
+import org.opencv.imgproc.Imgproc;
 
+import java.awt.*;
+import java.awt.image.BufferedImage;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -25,40 +40,243 @@ public class OcrUtils {
 
 
     /**
-     * 转换为FaceDetectedResult
+     * 转换为OcrBox
      * @param dt_boxes
      * @param img
      * @return
      */
-    public static DetectionResponse convertToDetectionResponse(NDList dt_boxes, Image img){
+    public static List<OcrBox> convertToOcrBox(NDList dt_boxes, Image img){
         if(Objects.isNull(dt_boxes) || dt_boxes.size() == 0){
             return null;
         }
-        DetectionResponse detectionResponse = new DetectionResponse();
-        List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
+        List<OcrBox> boxList = new ArrayList<OcrBox>();
         for(NDArray box : dt_boxes){
-            DetectionRectangle rectangle = new DetectionRectangle();
-            float[] points = box.toFloatArray();
-            log.info("points: {}", points);
-            int x = (int)points[0];
-            int y = (int)points[1];
-            int width = new BigDecimal(points[4]).subtract(new BigDecimal(points[6])).intValue();
-            int height = new BigDecimal(points[7]).subtract(new BigDecimal(points[1])).intValue();
-
-            // 修正边界，防止越界
-            if (x < 0) x = 0;
-            if (y < 0) y = 0;
-            if (x + width > img.getWidth()) width = img.getWidth() - x;
-            if (y + height > img.getHeight()) height = img.getHeight() - y;
-
-            rectangle.setX(x);
-            rectangle.setY(y);
-            rectangle.setHeight(height);
-            rectangle.setWidth(width);
-            detectionInfoList.add(new DetectionInfo(rectangle));
+            float[] pointsArr = box.toFloatArray();
+            //log.info("points: {}", pointsArr);
+            float[] lt = java.util.Arrays.copyOfRange(pointsArr, 0, 2);
+            float[] rt = java.util.Arrays.copyOfRange(pointsArr, 2, 4);
+            float[] rb = java.util.Arrays.copyOfRange(pointsArr, 4, 6);
+            float[] lb = java.util.Arrays.copyOfRange(pointsArr, 6, 8);
+            OcrBox ocrBox = new OcrBox(new Point(lt[0], lt[1]), new Point(rt[0], rt[1]), new Point(rb[0], rb[1]), new Point(lb[0], lb[1]));
+            boxList.add(ocrBox);
+        }
+        return boxList;
+    }
+
+    /**
+     * 欧式距离计算
+     *
+     * @param point1
+     * @param point2
+     * @return
+     */
+    public static float distance(float[] point1, float[] point2) {
+        float disX = point1[0] - point2[0];
+        float disY = point1[1] - point2[1];
+        float dis = (float) Math.sqrt(disX * disX + disY * disY);
+        return dis;
+    }
+
+    /**
+     * 图片旋转
+     *
+     * @param manager
+     * @param image
+     * @return
+     */
+    public static Image rotateImg(NDManager manager, Image image) {
+        NDArray rotated = NDImageUtils.rotate90(image.toNDArray(manager), 1);
+        return ImageFactory.getInstance().fromNDArray(rotated);
+    }
+
+    /**
+     * 逆时针旋转图片
+     *
+     * @param image
+     * @param times
+     * @return
+     */
+    public static Image rotateImg(Image image, int times) {
+        try (NDManager manager = NDManager.newBaseManager()) {
+            NDArray rotated = NDImageUtils.rotate90(image.toNDArray(manager), times);
+            return OpenCVImageFactory.getInstance().fromNDArray(rotated);
+        }
+    }
+
+
+    /**
+     * 逆时针旋转图片
+     *
+     * @param image
+     * @param angleEnum
+     * @return
+     */
+    public static Image rotateImg(Image image, AngleEnum angleEnum) {
+        try (NDManager manager = NDManager.newBaseManager()) {
+            int times = 0;
+            switch (angleEnum) {
+                case ANGLE_90:
+                    times = 1;
+                    break;
+                case ANGLE_180:
+                    times = 2;
+                    break;
+                case ANGLE_270:
+                    times = 3;
+                    break;
+            }
+            NDArray rotated = NDImageUtils.rotate90(image.toNDArray(manager), times);
+            return OpenCVImageFactory.getInstance().fromNDArray(rotated);
+        }
+    }
+
+
+    /**
+     * 转换为OcrInfo
+     * @param lines
+     * @return
+     */
+    public static OcrInfo convertToOcrInfo(List<ArrayList<RotatedBoxCompX>> lines){
+        if(Objects.isNull(lines) || lines.size() == 0){
+            return null;
+        }
+        List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
+        List<List<OcrItem>> lineList = new ArrayList<List<OcrItem>>();
+        String fullText = "";
+        for(ArrayList<RotatedBoxCompX> boxList : lines){
+            List<OcrItem> line = new ArrayList<OcrItem>();
+            for(RotatedBoxCompX box : boxList){
+                float[] pointsArr = box.getBox().toFloatArray();
+                float[] lt = java.util.Arrays.copyOfRange(pointsArr, 0, 2);
+                float[] rt = java.util.Arrays.copyOfRange(pointsArr, 2, 4);
+                float[] rb = java.util.Arrays.copyOfRange(pointsArr, 4, 6);
+                float[] lb = java.util.Arrays.copyOfRange(pointsArr, 6, 8);
+                OcrBox ocrBox = new OcrBox(new Point(lt[0], lt[1]), new Point(rt[0], rt[1]), new Point(rb[0], rb[1]), new Point(lb[0], lb[1]));
+                OcrItem ocrItem = new OcrItem(ocrBox, box.getText());
+                line.add(ocrItem);
+                String text = box.getText();
+                if(text.trim().equals(""))
+                    continue;
+                fullText += text + " ";
+            }
+            lineList.add(line);
+            fullText += '\n';
+        }
+        return new OcrInfo(lineList, fullText);
+    }
+
+
+    /**
+     * 放射变换+裁剪
+     * @param srcMat
+     * @param box
+     * @return
+     */
+    public static Image transformAndCrop(Mat srcMat, OcrBox box){
+        float[] pointsArr = box.toFloatArray();
+        float[] lt = java.util.Arrays.copyOfRange(pointsArr, 0, 2);
+        float[] rt = java.util.Arrays.copyOfRange(pointsArr, 2, 4);
+        float[] rb = java.util.Arrays.copyOfRange(pointsArr, 4, 6);
+        float[] lb = java.util.Arrays.copyOfRange(pointsArr, 6, 8);
+        int img_crop_width = (int) Math.max(OcrUtils.distance(lt, rt), OcrUtils.distance(rb, lb));
+        int img_crop_height = (int) Math.max(OcrUtils.distance(lt, lb), OcrUtils.distance(rt, rb));
+        List<ai.djl.modality.cv.output.Point> srcPoints = new ArrayList<>();
+        srcPoints.add(new ai.djl.modality.cv.output.Point(lt[0], lt[1]));
+        srcPoints.add(new ai.djl.modality.cv.output.Point(rt[0], rt[1]));
+        srcPoints.add(new ai.djl.modality.cv.output.Point(rb[0], rb[1]));
+        srcPoints.add(new ai.djl.modality.cv.output.Point(lb[0], lb[1]));
+        List<ai.djl.modality.cv.output.Point> dstPoints = new ArrayList<>();
+        dstPoints.add(new ai.djl.modality.cv.output.Point(0, 0));
+        dstPoints.add(new ai.djl.modality.cv.output.Point(img_crop_width, 0));
+        dstPoints.add(new ai.djl.modality.cv.output.Point(img_crop_width, img_crop_height));
+        dstPoints.add(new ai.djl.modality.cv.output.Point(0, img_crop_height));
+        Mat srcPoint2f = OcrNDArrayUtils.toMat(srcPoints);
+        Mat dstPoint2f = OcrNDArrayUtils.toMat(dstPoints);
+        //透视变换
+        Mat cvMat = OcrOpenCVUtils.perspectiveTransform(srcMat, srcPoint2f, dstPoint2f);
+        Image subImg = OpenCVImageFactory.getInstance().fromImage(cvMat);
+        //ImageUtils.saveImage(subImg, i + ".png", "build/output");
+        //变换后裁剪
+        subImg = subImg.getSubImage(0, 0, img_crop_width, img_crop_height);
+        cvMat.release();
+        srcPoint2f.release();
+        dstPoint2f.release();
+        return subImg;
+    }
+
+    /**
+     * 绘制文本框
+     *
+     * @param mat
+     * @param boxList
+     */
+    public static void drawRect(Mat mat, List<OcrBox> boxList) {
+        for(OcrBox ocrBox : boxList){
+            Imgproc.line(mat, ocrBox.getTopLeft().toCvPoint(), ocrBox.getTopRight().toCvPoint(), new Scalar(0, 255, 0), 1);
+            Imgproc.line(mat, ocrBox.getTopRight().toCvPoint(), ocrBox.getBottomRight().toCvPoint(), new Scalar(0, 255, 0),1);
+            Imgproc.line(mat, ocrBox.getBottomRight().toCvPoint(), ocrBox.getBottomLeft().toCvPoint(), new Scalar(0, 255, 0),1);
+            Imgproc.line(mat, ocrBox.getBottomLeft().toCvPoint(), ocrBox.getTopLeft().toCvPoint(), new Scalar(0, 255, 0), 1);
+        }
+    }
+
+
+    /**
+     * 绘制文本框及文本
+     * @param image
+     * @param ocrInfo
+     */
+    public static void drawRectWithText(BufferedImage image, OcrInfo ocrInfo,  int fontSize) {
+        // 将绘制图像转换为Graphics2D
+        Graphics2D g = (Graphics2D) image.getGraphics();
+        try {
+            Font font = new Font("楷体", Font.PLAIN, fontSize);
+            g.setFont(font);
+            g.setColor(new Color(0, 0, 255));
+            // 声明画笔属性 ：粗 细（单位像素）末端无修饰 折线处呈尖角
+            BasicStroke bStroke = new BasicStroke(2, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER);
+            g.setStroke(bStroke);
+            for(List<OcrItem> ocrItemList : ocrInfo.getLineList()){
+                for(OcrItem item : ocrItemList){
+                    OcrBox box = item.getOcrBox();
+                    int[] xPoints = {
+                            (int)box.getTopLeft().getX(),
+                            (int)box.getTopRight().getX(),
+                            (int)box.getBottomRight().getX(),
+                            (int)box.getBottomLeft().getX(),
+                            (int)box.getTopLeft().getX()
+                    };
+                    int[] yPoints = {
+                            (int)box.getTopLeft().getY(),
+                            (int)box.getTopRight().getY(),
+                            (int)box.getBottomRight().getY(),
+                            (int)box.getBottomLeft().getY(),
+                            (int)box.getTopLeft().getY()
+                    };
+                    g.drawPolyline(xPoints, yPoints, 5);
+                    g.drawString(item.getText(), xPoints[0], yPoints[0]);
+                }
+            }
+        } finally {
+            g.dispose();
+        }
+    }
+
+
+    /**
+     * 绘制文本框及文本
+     * @param srcMat
+     * @param itemList
+     */
+    public static void drawRectWithText(Mat srcMat, List<OcrItem> itemList) {
+        for(OcrItem item : itemList){
+            OcrBox ocrBox = item.getOcrBox();
+            Imgproc.line(srcMat, ocrBox.getTopLeft().toCvPoint(), ocrBox.getTopRight().toCvPoint(), new Scalar(0, 255, 0), 1);
+            Imgproc.line(srcMat, ocrBox.getTopRight().toCvPoint(), ocrBox.getBottomRight().toCvPoint(), new Scalar(0, 255, 0),1);
+            Imgproc.line(srcMat, ocrBox.getBottomRight().toCvPoint(), ocrBox.getBottomLeft().toCvPoint(), new Scalar(0, 255, 0),1);
+            Imgproc.line(srcMat, ocrBox.getBottomLeft().toCvPoint(), ocrBox.getTopLeft().toCvPoint(), new Scalar(0, 255, 0), 1);
+            // 中文乱码
+            Imgproc.putText(srcMat, item.getAngle().getValue(), ocrBox.getTopLeft().toCvPoint(), Imgproc.FONT_HERSHEY_SCRIPT_SIMPLEX, 1.0, new Scalar(0, 255, 0), 1);
         }
-        detectionResponse.setDetectionInfoList(detectionInfoList);
-        return detectionResponse;
     }