Ver Fonte

图片查重

qin há 1 ano atrás
pai
commit
83f29b0a2a

+ 1 - 1
src/main/java/com/poteviohealth/cgp/statistics/mapper/DistinctionMapper.java

@@ -17,7 +17,7 @@ import java.util.List;
 public interface DistinctionMapper extends BaseMapper<Distinction>{
 
     @SqlParser(filter = true)
-    DistinctionDTO checkDis(@Param("operatorId")Integer operatorId, @Param("dbName") String dbName, @Param("uid") String uid, @Param("dCount") Integer dCount, @Param("date") String date);
+    DistinctionDTO checkDis(@Param("operatorId")Integer operatorId, @Param("dbName") String dbName, @Param("uid1") String uid1,@Param("uid2") String uid2,@Param("uid3") String uid3,@Param("uid4") String uid4,@Param("uid5") String uid5,@Param("uid6") String uid6,@Param("uid7") String uid7,@Param("uid8") String uid8,@Param("uid9") String uid9, @Param("dCount") Integer dCount, @Param("date") String date);
 
     @SqlParser(filter = true)
     void insertBatch(@Param("list") List<Distinction> list);

+ 20 - 0
src/main/java/com/poteviohealth/cgp/statistics/model/Distinction.java

@@ -1,5 +1,6 @@
 package com.poteviohealth.cgp.statistics.model;
 
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.poteviohealth.cgp.common.model.BaseEntity;
 import io.swagger.annotations.ApiModelProperty;
@@ -28,8 +29,27 @@ public class Distinction extends BaseEntity {
     private Long orderId;
     @ApiModelProperty(value = "上传时间")
     private Date uploadDate;
+    @TableField(exist = false)
     @ApiModelProperty(value = "图片标识")
     private String uid;
+    @ApiModelProperty(value = "图片标识")
+    private String uid1;
+    @ApiModelProperty(value = "图片标识")
+    private String uid2;
+    @ApiModelProperty(value = "图片标识")
+    private String uid3;
+    @ApiModelProperty(value = "图片标识")
+    private String uid4;
+    @ApiModelProperty(value = "图片标识")
+    private String uid5;
+    @ApiModelProperty(value = "图片标识")
+    private String uid6;
+    @ApiModelProperty(value = "图片标识")
+    private String uid7;
+    @ApiModelProperty(value = "图片标识")
+    private String uid8;
+    @ApiModelProperty(value = "图片标识")
+    private String uid9;
     @ApiModelProperty(value = "图片路径")
     private String url;
 }

+ 60 - 5
src/main/java/com/poteviohealth/cgp/statistics/service/impl/DistinctionServiceImpl.java

@@ -9,7 +9,7 @@ import com.poteviohealth.cgp.common.model.VaultsResponse;
 import com.poteviohealth.cgp.common.service.impl.BaseServiceImpl;
 import com.poteviohealth.cgp.common.utils.CgpTool;
 import com.poteviohealth.cgp.common.utils.DateUtils;
-import com.poteviohealth.cgp.common.utils.ImageUtils;
+import com.poteviohealth.cgp.common.utils.FingerImage;
 import com.poteviohealth.cgp.statistics.mapper.DistinctionMapper;
 import com.poteviohealth.cgp.statistics.model.Distinction;
 import com.poteviohealth.cgp.statistics.model.outdto.DistinctionDTO;
@@ -43,7 +43,7 @@ public class DistinctionServiceImpl extends BaseServiceImpl<DistinctionMapper, D
     @Override
     public DistinctionDTO checkImg(File file, Long orderId, List<String> uidList){
         DistinctionDTO dto = new DistinctionDTO();
-        String uid = ImageUtils.readImagePix(file);
+        String uid = FingerImage.readImagePix(file);
         dto.setUid(uid);
         if("0".equals(orderCacheVal(CgpTool.CACHE_ORDER_PHOTO_CHECK))){
             return dto;
@@ -51,10 +51,10 @@ public class DistinctionServiceImpl extends BaseServiceImpl<DistinctionMapper, D
 
         Integer day = Integer.parseInt(orderCacheVal(CgpTool.CACHE_ORDER_PHOTO_DAY));
         Double threshold = Double.parseDouble(orderCacheVal(CgpTool.CACHE_ORDER_PHOTO_THRESHOLD));
-        int count = ImageUtils.different(threshold);
+        int count = FingerImage.different(threshold);
         boolean isCheck = false;
         for (String s : uidList) {
-            if(ImageUtils.similarity(s,uid)<=count){
+            if(FingerImage.similarity(s,uid)<=count){
                 isCheck = true;
                 break;
             }
@@ -63,7 +63,34 @@ public class DistinctionServiceImpl extends BaseServiceImpl<DistinctionMapper, D
         if(isCheck){
             dto.setOrderId(orderId);
         }else{
-            DistinctionDTO dis = baseMapper.checkDis(TokenContext.cureOperatorId(),dbName,uid,count, DateFormatUtils.format(DateUtils.caculDays(new Date(), -day),"yyyy-MM-dd HH:mm:ss"));
+            final int SIZE = 64;
+            int length = uid.length();
+            String uid1 ="",uid2 ="",uid3 ="",uid4 ="",uid5 ="",uid6 ="",uid7 ="",uid8 ="",uid9 ="";
+            if(length % SIZE == 0) {
+                for (int i = 1; i <= length / SIZE; i++) {
+                    String val = uid.substring((i - 1) * SIZE, (i * SIZE) - 1);
+                    if (i == 1) {
+                        uid1 = val;
+                    } else if (i == 2) {
+                        uid2 = val;
+                    } else if (i == 3) {
+                        uid3 = val;
+                    } else if (i == 4) {
+                        uid4 = val;
+                    } else if (i == 5) {
+                        uid5 = val;
+                    } else if (i == 6) {
+                        uid6 = val;
+                    } else if (i == 7) {
+                        uid7 = val;
+                    } else if (i == 8) {
+                        uid8 = val;
+                    } else if (i == 9) {
+                        uid9 = val;
+                    }
+                }
+            }
+            DistinctionDTO dis = baseMapper.checkDis(TokenContext.cureOperatorId(),dbName,uid1,uid2,uid3,uid4,uid5,uid6,uid7,uid8,uid9,count, DateFormatUtils.format(DateUtils.caculDays(new Date(), -day),"yyyy-MM-dd HH:mm:ss"));
             if(dis != null){
                 dto.setOrderId(dis.getOrderId());
                 dto.setRurl(dis.getRurl());
@@ -74,6 +101,34 @@ public class DistinctionServiceImpl extends BaseServiceImpl<DistinctionMapper, D
 
     @Override
     public void insertBatch(List<Distinction> list) {
+        final int SIZE = 64;
+        for (Distinction distinction : list) {
+            int length = distinction.getUid().length();
+            if(length % SIZE == 0) {
+                for (int i = 1; i <= length / SIZE; i++) {
+                    String val = distinction.getUid().substring((i - 1) * SIZE, (i * SIZE) - 1);
+                    if (i == 1) {
+                        distinction.setUid1(val);
+                    } else if (i == 2) {
+                        distinction.setUid2(val);
+                    } else if (i == 3) {
+                        distinction.setUid3(val);
+                    } else if (i == 4) {
+                        distinction.setUid4(val);
+                    } else if (i == 5) {
+                        distinction.setUid5(val);
+                    } else if (i == 6) {
+                        distinction.setUid6(val);
+                    } else if (i == 7) {
+                        distinction.setUid7(val);
+                    } else if (i == 8) {
+                        distinction.setUid8(val);
+                    } else if (i == 9) {
+                        distinction.setUid9(val);
+                    }
+                }
+            }
+        }
         baseMapper.insertBatch(list);
     }
 

+ 314 - 0
src/main/java/com/poteviohealth/cgp/statistics/utils/FingerPrint.java

@@ -0,0 +1,314 @@
+package com.poteviohealth.cgp.statistics.utils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.color.ColorSpace;
+import java.awt.image.BufferedImage;
+import java.awt.image.ColorConvertOp;
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.Arrays;
+
+public class FingerPrint {
+
+    /**
+     * 图像指纹的尺寸,将图像resize到指定的尺寸,来计算哈希数组
+     */
+    private static final int HASH_SIZE=24;
+    /**
+     * 保存图像指纹的二值化矩阵
+     */
+    private final byte[] binaryzationMatrix;
+
+    public FingerPrint(byte[] hashValue) {
+        this.binaryzationMatrix=hashValue;
+    }
+    public FingerPrint(String hashValue) {
+        this(toBytes(hashValue));
+    }
+    public FingerPrint (BufferedImage src){
+        this(hashValue(src));
+    }
+    private static byte[] hashValue(BufferedImage src){
+        BufferedImage hashImage = resize(src,HASH_SIZE,HASH_SIZE);
+        byte[] matrixGray = (byte[]) toGray(hashImage).getData().getDataElements(0, 0, HASH_SIZE, HASH_SIZE, null);
+        return  binaryzation(matrixGray);
+    }
+    /**
+     * 从压缩格式指纹创建{@link FingerPrint}对象
+     * @param compactValue
+     * @return
+     */
+    public static FingerPrint createFromCompact(byte[] compactValue){
+        return new FingerPrint(uncompact(compactValue));
+    }
+
+    public static boolean validHashValue(byte[] hashValue){
+        if(hashValue.length!=HASH_SIZE)
+            return false;
+        for(byte b:hashValue){
+            if(0!=b&&1!=b)return false;
+        }
+        return true;
+    }
+    public static boolean validHashValue(String hashValue){
+        if(hashValue.length()!=HASH_SIZE)
+            return false;
+        for(int i=0;i<hashValue.length();++i){
+            if('0'!=hashValue.charAt(i)&&'1'!=hashValue.charAt(i))return false;
+        }
+        return true;
+    }
+    public byte[] compact(){
+        return compact(binaryzationMatrix);
+    }
+
+    /**
+     * 指纹数据按位压缩
+     * @param hashValue
+     * @return
+     */
+    public static byte[] compact(byte[] hashValue){
+        byte[] result=new byte[(hashValue.length+7)>>3];
+        byte b=0;
+        for(int i=0;i<hashValue.length;++i){
+            if(0==(i&7)){
+                b=0;
+            }
+            if(1==hashValue[i]){
+                b|=1<<(i&7);
+            }else if(hashValue[i]!=0)
+                throw new IllegalArgumentException("invalid hashValue,every element must be 0 or 1");
+            if(7==(i&7)||i==hashValue.length-1){
+                result[i>>3]=b;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * 压缩格式的指纹解压缩
+     * @param compactValue
+     * @return
+     */
+    public static byte[] uncompact(byte[] compactValue){
+        byte[] result=new byte[compactValue.length<<3];
+        for(int i=0;i<result.length;++i){
+            if((compactValue[i>>3]&(1<<(i&7)))==0)
+                result[i]=0;
+            else
+                result[i]=1;
+        }
+        return result;
+    }
+    /**
+     * 字符串类型的指纹数据转为字节数组
+     * @param hashValue
+     * @return
+     */
+    private static byte[] toBytes(String hashValue){
+        hashValue=hashValue.replaceAll("\\s", "");
+        byte[] result=new byte[hashValue.length()];
+        for(int i=0;i<result.length;++i){
+            char c = hashValue.charAt(i);
+            if('0'==c)
+                result[i]=0;
+            else if('1'==c)
+                result[i]=1;
+            else
+                throw new IllegalArgumentException("invalid hashValue String");
+        }
+        return result;
+    }
+    /**
+     * 缩放图像到指定尺寸
+     * @param src
+     * @param width
+     * @param height
+     * @return
+     */
+    private static BufferedImage resize(Image src,int width,int height){
+        BufferedImage result = new BufferedImage(width, height,
+                BufferedImage.TYPE_3BYTE_BGR);
+        Graphics g = result.getGraphics();
+        try{
+            g.drawImage(src.getScaledInstance(width, height, Image.SCALE_SMOOTH), 0, 0, null);
+        }finally{
+            g.dispose();
+        }
+        return result;
+    }
+    /**
+     * 计算均值
+     * @param src
+     * @return
+     */
+    private static  int mean(byte[] src){
+        long sum=0;
+        // 将数组元素转为无符号整数
+        for(byte b:src)sum+=(long)b&0xff;
+        return (int) (Math.round((float)sum/src.length));
+    }
+    /**
+     * 二值化处理
+     * @param src
+     * @return
+     */
+    private static byte[] binaryzation(byte[]src){
+        byte[] dst = src.clone();
+        int mean=mean(src);
+        for(int i=0;i<dst.length;++i){
+            // 将数组元素转为无符号整数再比较
+            dst[i]=(byte) (((int)dst[i]&0xff)>=mean?1:0);
+        }
+        return dst;
+
+    }
+    /**
+     * 转灰度图像
+     * @param src
+     * @return
+     */
+    private static BufferedImage toGray(BufferedImage src){
+        if(src.getType()==BufferedImage.TYPE_BYTE_GRAY){
+            return src;
+        }else{
+            // 图像转灰
+            BufferedImage grayImage = new BufferedImage(src.getWidth(), src.getHeight(),
+                    BufferedImage.TYPE_BYTE_GRAY);
+            new ColorConvertOp(ColorSpace.getInstance(ColorSpace.CS_GRAY), null).filter(src, grayImage);
+            return grayImage;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return toString(true);
+    }
+    /**
+     * @param multiLine 是否分行
+     * @return
+     */
+    public String toString(boolean multiLine) {
+        StringBuffer buffer=new StringBuffer();
+        int count=0;
+        for(byte b:this.binaryzationMatrix){
+            buffer.append(0==b?'0':'1');
+            if(multiLine&&++count%HASH_SIZE==0)
+                buffer.append('\n');
+        }
+        return buffer.toString();
+    }
+
+    public String toString(byte [] val) {
+        StringBuffer buffer=new StringBuffer();
+        for(byte b:val){
+            buffer.append(0==b?'0':'1');
+        }
+        return buffer.toString();
+    }
+    @Override
+    public boolean equals(Object obj) {
+        if(obj instanceof FingerPrint){
+            return Arrays.equals(this.binaryzationMatrix,((FingerPrint)obj).binaryzationMatrix);
+        }else
+            return super.equals(obj);
+    }
+
+    /**
+     * 与指定的压缩格式指纹比较相似度
+     * @param compactValue
+     * @return
+     * @see #compare(FingerPrint)
+     */
+    public float compareCompact(byte[] compactValue){
+        return compare(createFromCompact(compactValue));
+    }
+    /**
+     * @param hashValue
+     * @return
+     * @see #compare(FingerPrint)
+     */
+    public float compare(String hashValue){
+        return compare(new FingerPrint(hashValue));
+    }
+    /**
+     * 与指定的指纹比较相似度
+     * @param hashValue
+     * @return
+     * @see #compare(FingerPrint)
+     */
+    public float compare(byte[] hashValue){
+        return compare(new FingerPrint(hashValue));
+    }
+    /**
+     * 与指定图像比较相似度
+     * @param image2
+     * @return
+     * @see #compare(FingerPrint)
+     */
+    public float compare(BufferedImage image2){
+        return compare(new FingerPrint(image2));
+    }
+    /**
+     * 比较指纹相似度
+     * @param src
+     * @return
+     * @see #compare(byte[], byte[])
+     */
+    public float compare(FingerPrint src){
+        return compare(binaryzationMatrix,src.binaryzationMatrix);
+    }
+    /**
+     * 判断两个数组相似度,数组长度必须一致否则抛出异常
+     * @param f1
+     * @param f2
+     * @return 返回相似度(0.0~1.0)
+     */
+    private static float compare(byte[] f1,byte[] f2){
+        int sameCount=0;
+        for(int i=0;i<f1.length;++i){
+            if(f1[i]==f2[i])++sameCount;
+        }
+        System.out.println(f1.length);
+        System.out.println(sameCount);
+        return (float)sameCount/f1.length;
+    }
+
+    public static float compareString(String val1,String val2){
+        BigInteger num1= new BigInteger(val1, 2);
+        BigInteger  num2= new BigInteger(val2, 2);
+        BigInteger difference = num1.xor(num2);
+        System.out.println(num1);
+        System.out.println(num2);
+        System.out.println(difference.bitCount());
+        return (float)(val1.length()-difference.bitCount())/val1.length();
+    }
+    public static float compareCompact(byte[] f1,byte[] f2){
+        return compare(uncompact(f1),uncompact(f2));
+    }
+    public static float compare(BufferedImage image1, BufferedImage image2){
+        return new FingerPrint(image1).compare(new FingerPrint(image2));
+    }
+
+    public static void main(String[] args) {
+        FingerPrint fp1 = null;
+        try {
+            fp1 = new FingerPrint(ImageIO.read(new File("C:\\Users\\86180\\Desktop\\333.jpg")));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        FingerPrint fp2 = null;
+        try {
+            fp2 = new FingerPrint(ImageIO.read(new File("C:\\Users\\86180\\Desktop\\222.jpg")));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        System.out.println(fp1.toString(false));
+        System.out.println(fp2.toString(false));
+        System.out.println(compareString(fp1.toString(false),fp2.toString(false)));
+        //System.out.println(fp1.toString(true));
+        System.out.printf("sim=%f",fp1.compare(fp2));
+    }
+}

+ 3 - 2
src/main/java/com/poteviohealth/cgp/statistics/utils/OssUtils.java

@@ -4,6 +4,7 @@ import com.aliyun.oss.OSS;
 import com.aliyun.oss.OSSClientBuilder;
 import com.poteviohealth.cgp.common.model.VaultsResponse;
 import com.poteviohealth.cgp.common.utils.CgpTool;
+import com.poteviohealth.cgp.common.utils.FingerImage;
 import com.poteviohealth.cgp.common.utils.ImageUtils;
 import lombok.extern.log4j.Log4j2;
 import org.apache.commons.io.FileUtils;
@@ -183,12 +184,12 @@ public class OssUtils {
                 FileUtils.copyInputStreamToFile(inputStream, tempFile);
                 File file = this.generateCover(tempFile);
                 tempFile.delete();
-                String val = ImageUtils.readImagePix(file);
+                String val = FingerImage.readImagePix(file);
                 file.delete();
                 return val;
             }
 
-            return ImageUtils.readImagePix(inputStream);
+            return FingerImage.readImagePix(inputStream);
         } catch (Exception e) {
             e.printStackTrace();
             return null;

+ 32 - 3
src/main/resources/mapper/statistics/DistinctionMapper.xml

@@ -2,9 +2,9 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.poteviohealth.cgp.statistics.mapper.DistinctionMapper">
     <insert id="insertBatch">
-        INSERT INTO f_distinction_img (created_date,modified_date,yn,operator_id,supplier_id,station_id,order_id,upload_date,uid,url) VALUES
+        INSERT INTO f_distinction_img (created_date,modified_date,yn,operator_id,supplier_id,station_id,order_id,upload_date,url,uid1,uid2,uid3,uid4,uid5,uid6,uid7,uid8,uid9) VALUES
         <foreach collection="list" item="d" separator=",">
-            (NOW(),NOW(),0,#{d.operatorId},#{d.supplierId},#{d.stationId},#{d.orderId},NOW(),#{d.uid},#{d.url})
+            (NOW(),NOW(),0,#{d.operatorId},#{d.supplierId},#{d.stationId},#{d.orderId},NOW(),#{d.url},b#{d.uid1},b#{d.uid2},b#{d.uid3},b#{d.uid4},b#{d.uid5},b#{d.uid6},b#{d.uid7},b#{d.uid8},b#{d.uid9})
         </foreach>
     </insert>
     <select id="checkDis" resultType="com.poteviohealth.cgp.statistics.model.outdto.DistinctionDTO">
@@ -12,6 +12,35 @@
         where yn = 0
         and operator_id = #{operatorId}
         and upload_date >= #{date}
-        and BIT_COUNT(uid ^ #{uid} )&lt;=#{dCount} LIMIT 1
+        and
+        <if test="uid1 != '' ">
+            BIT_COUNT(uid1 ^ b#{uid1} )+
+        </if>
+        <if test="uid2 != '' ">
+            BIT_COUNT(uid2 ^ b#{uid2} )+
+        </if>
+        <if test="uid3 != '' ">
+            BIT_COUNT(uid3 ^ b#{uid3} )+
+        </if>
+        <if test="uid4 != '' ">
+            BIT_COUNT(uid4 ^ b#{uid4} )+
+        </if>
+        <if test="uid5 != '' ">
+            BIT_COUNT(uid5 ^ b#{uid5} )+
+        </if>
+        <if test="uid6 != '' ">
+            BIT_COUNT(uid6 ^ b#{uid6} )+
+        </if>
+        <if test="uid7 != '' ">
+            BIT_COUNT(uid7 ^ b#{uid7} )+
+        </if>
+        <if test="uid8 != '' ">
+            BIT_COUNT(uid8 ^ b#{uid8} )+
+        </if>
+        <if test="uid9 != '' ">
+            BIT_COUNT(uid9 ^ b#{uid9} )+
+        </if>
+        0
+            &lt;=#{dCount} LIMIT 1
     </select>
 </mapper>