opencv - Finding if two images are similar

Question

I'm trying to find out it two images are similar with openCV using image matching. I'm running the following code:

public static void match(String firstImage, String secondImage, String outputFile) {

    FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
    DescriptorExtractor descriptor = DescriptorExtractor.create(DescriptorExtractor.ORB);
    DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);


    Mat firstImg = Imgcodecs.imread(firstImage, Imgcodecs.CV_LOAD_IMAGE_GRAYSCALE);
    MatOfKeyPoint firstKeypoints = new MatOfKeyPoint();
    Mat firstDescriptors = new Mat();
    detector.detect(firstImg, firstKeypoints);
    descriptor.compute(firstImg, firstKeypoints, firstDescriptors);

    Mat secondImg = Imgcodecs.imread(secondImage,      Imgcodecs.CV_LOAD_IMAGE_GRAYSCALE);
    MatOfKeyPoint secondKeypoints = new MatOfKeyPoint();
    Mat secondDescriptors = new Mat();
    detector.detect(secondImg, secondKeypoints);
    descriptor.compute(secondImg, secondKeypoints, secondDescriptors);

    MatOfDMatch matches = new MatOfDMatch();
    matcher.match(firstDescriptors, secondDescriptors, matches);

    float minDis = Float.MAX_VALUE;
    for (int i = 0;i < matches.rows();i++) {
        if (matches.toArray()[i].distance < minDis)
            minDis = matches.toArray()[i].distance;
    }
    LinkedList<DMatch> goodMatches = new LinkedList<>();
    for (int i = 0;i < matches.rows();i++) {
        if (matches.toArray()[i].distance < minDis*3)
            goodMatches.add(matches.toArray()[i]);
    }


    List<Point> pts1 = new ArrayList<Point>();
    List<Point> pts2 = new ArrayList<Point>();
    for(int i = 0; i<goodMatches.size(); i++){
        pts1.add(firstKeypoints.toList().get(goodMatches.get(i).queryIdx).pt);
        pts2.add(secondKeypoints.toList().get(goodMatches.get(i).trainIdx).pt);
    }

    // convertion of data types - there is maybe a more beautiful way
    Mat outputMask = new Mat();
    MatOfPoint2f pts1Mat = new MatOfPoint2f();
    pts1Mat.fromList(pts1);
    MatOfPoint2f pts2Mat = new MatOfPoint2f();
    pts2Mat.fromList(pts2);

    Calib3d.findHomography(pts1Mat, pts2Mat, Calib3d.RANSAC, 15, outputMask, 2000, 0.995);

    // outputMask contains zeros and ones indicating which matches are filtered
    LinkedList<DMatch> betterMatches = new LinkedList<DMatch>();
    for (int i = 0; i < goodMatches.size(); i++) {
        if (outputMask.get(i, 0)[0] != 0.0) {
            betterMatches.add(goodMatches.get(i));
        }
    }

    Mat outputImg = new Mat();
    MatOfDMatch betterMatchesMat = new MatOfDMatch();
    betterMatchesMat.fromList(betterMatches);
    Features2d.drawMatches(firstImg, firstKeypoints, secondImg, secondKeypoints, betterMatchesMat, outputImg);
    Imgcodecs.imwrite(outputFile, outputImg);
}

When the images are similar the result looks like:

When the images are not similar the result looks like:

You can see that in the first case the match lines are parallel so it makes sense that the two images are similar. In the second case the match lines are not parallel so it makes sense that the images are not similar. Is there a standard way to analyse these matches and to find in which case the images are most probably similar?

Answer 1

For checking the appropriateness of homography matrix between two perspectives of similar images, I wrote this function. You can choose what you prefer out of the possibilities, depending on which one you find more suitable:

 private static boolean check_homography(Mat homography_mat){
    /* Check 1. Compute the determinant of the homography, and see if it's too close 
     to zero for comfort*/
    if(!homography_mat.empty())
    {
        double Determinant = Core.determinant(homography_mat);
        if (Determinant > 0.1)
            return true;
        else 
            return false;
    }
    else 
        return false;

    /* Check 2. Compute its SVD, and verify that the ratio of the first-to-last 
     singular value is not too high (order of 1.0E7). */   
    Mat singularValues = new Mat();
    Core.SVDecomp(homography_mat, singularValues, new Mat(), new Mat(), Core.SVD_NO_UV);

    System.out.print("
 Printing the singular values of the homography");
    for (int i = 0; i < singularValues.rows(); i++){
        for ( int j = 0; j < singularValues.cols(); j++){
            System.out.print("
 Element at ( " + i + ", " + j + " ) is " + singularValues.get(i, j)[0]);
        }
    }
    double conditionNumber = singularValues.get(0, 0)[0] / singularValues.get(2, 0)[0];
    System.out.print("
 Condition number is : " + conditionNumber);

    if(conditionNumber < Math.pow(10, 7)){
        System.out.print("
 Homography matrix is non-singular");
        return true;
        }
    else{
        System.out.print("
 Homography matrix is singular (or very close)");
        return false;
        }
    /* Check 3. Check the compare absolute values at (0,0) and (0,1) with (1,1) and (1,0) 
     * respectively. If the two differences are close to 0, the homography matrix is 
     * good. (This just takes of rotation and not translation)
     * */
    if(Math.abs((Math.abs(homography_mat.get(0, 0)[0]) - Math.abs(homography_mat.get(1, 1)[0]))) <= 0.1){
        if(Math.abs((Math.abs(homography_mat.get(0, 1)[0]) - Math.abs(homography_mat.get(1, 0)[0]))) <= 0.1){
            System.out.print("
 The homography matrix is good");
            return true;
        }
    }
        else{
            System.out.print("
 The homography matrix is bad");
            return false;
        }
    return false;
    /*
     * Check 4: If the determinant of the top-left 2 by 2 matrix (rotation) > 0, transformation is orientation
     * preserving.
     * Else if the determinant is < 0, it is orientation reversing
     * 
     * */
     Determinant of the rotation mat
    double det = homography_mat.get(0, 0)[0] * homography_mat.get(1,1)[0] - homography_mat.get(0, 1)[0] * homography_mat.get(1, 0)[0];
    if (det < 0)
        return false;

    double N1 = Math.sqrt(homography_mat.get(0, 0)[0] * homography_mat.get(0, 0)[0] + homography_mat.get(1, 0)[0] * homography_mat.get(1, 0)[0]);
    if (N1 > 4 || N1 < 0.1)
        return false;

    double N2 = Math.sqrt(homography_mat.get(0, 1)[0] * homography_mat.get(0, 1)[0] + homography_mat.get(1, 1)[0] * homography_mat.get(1, 1)[0]);
    if (N2 > 4 || N2 < 0.1)
        return false;

    double N3 = Math.sqrt(homography_mat.get(2, 0)[0] * homography_mat.get(2, 0)[0] + homography_mat.get(2,1)[0] * homography_mat.get(2, 1)[0]);
    if (N3 < 0.002)
        return false;

    return true;

}

Note - I coded this for Java, OpenCV when using ORB. I personally (with experience I guess), can look at the Homography matrix and say more or less if its good or not and hence the Check 1. Hope it helps!!

EDIT Additionally, as @Micka mentioned, this is subject to your preference, you can also iterate along the two images pixel by pixel to find the similarity. The method I post here is used to check if the homography is good or not, however, it should be noted that you can get bad homography even when two images are similar depending on the images and the method(pre/post processing, descriptor, detector etc. )