import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
@Override
    public CharSequence getAdditionalFields(String filename) {
        StringBuilder sb = new StringBuilder(1024);
        int fileNumber = Integer.parseInt(filename.substring(filename.lastIndexOf("\\")).replaceAll("[^0-9]", "")) - 1;
        String tagFileName = "D:\\DataSets\\MirFlickr\\tags_raw\\" + (fileNumber/10000) + "\\" + fileNumber + ".txt";
//        String tagFileName = filename.substring(0, filename.lastIndexOf("\\")) + "\\" + fileNumber + ".txt";
        try {
            sb.append("<field name=\"tags\">");
            StringBuilder tmp = new StringBuilder();
            FileUtils.readWholeFile(new File(tagFileName), tmp);
            sb.append(tmp.toString().replaceAll("\\s", " "));  // replacing \n\r\t and all those with " "
            sb.append("</field>");
        } catch (IOException e) {
            e.printStackTrace();
            return "";
        }
        return sb;
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testIndex() throws IOException {
        // indexing
        System.out.println("-< Getting files to index >--------------");
        ArrayList<String> images = FileUtils.getAllImages(new File(testExtensive), true);
        System.out.println("-< Indexing " + images.size() + " files >--------------");
        indexFiles(images, builder, indexPath);

//        in case of "bag of visual words" ...
//        SiftFeatureHistogramBuilder sh1 = new SiftFeatureHistogramBuilder(IndexReader.open(FSDirectory.open(new File(indexPath))), 200, 8000);
//        sh1.index();
//        SurfFeatureHistogramBuilder sh = new SurfFeatureHistogramBuilder(IndexReader.open(FSDirectory.open(new File(indexPath))), 200, 8000);
//        sh.index();
//        MSERFeatureHistogramBuilder sh = new MSERFeatureHistogramBuilder(IndexReader.open(FSDirectory.open(new File(indexPath))), 200, 8000);
//        sh.index();

        System.out.println("-< Indexing finished >--------------");

    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
private void indexFiles(String dir, String index, int featureIndex, boolean createNewIndex) throws IOException {
        ArrayList<String> images = FileUtils.getAllImages(new File(dir), true);
        IndexWriter iw = LuceneUtils.createIndexWriter(index, createNewIndex, LuceneUtils.AnalyzerType.WhitespaceAnalyzer);
        // select one feature for the large index:
        int count = 0;
        long ms = System.currentTimeMillis();
        DocumentBuilder builder = new ChainedDocumentBuilder();
        ((ChainedDocumentBuilder) builder).addBuilder(builders[featureIndex]);
//        ((ChainedDocumentBuilder) builder).addBuilder(builders[0]);
        for (Iterator<String> iterator = images.iterator(); iterator.hasNext(); ) {
            count++;
            if (count > 100 && count % 5000 == 0) {
                System.out.println(count + " files indexed. " + (System.currentTimeMillis() - ms) / (count) + " ms per file");
            }
            String file = iterator.next();
            try {
                iw.addDocument(builder.createDocument(new FileInputStream(file), file));
            } catch (Exception e) {
                System.err.println("Error: " + e.getMessage());
            }
        }
        iw.close();
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testSearchIndexLarge() throws IOException {

        for (int i = 0; i < 10; i++) {
            int queryDocID = (int) (Math.random() * 800);
//            queryDocID = 877 * (i + 1);
            IndexReader reader = DirectoryReader.open(FSDirectory.open(new File("index-large")));
            // select one feature for the large index:
            int featureIndex = 13;
            int count = 0;
            long ms = System.currentTimeMillis();
            ImageSearchHits hits = searchers[featureIndex].search(reader.document(queryDocID), reader);
            for (int j = 0; j < hits.length(); j++) {
                String fileName = hits.doc(j).getValues(
                        DocumentBuilder.FIELD_NAME_IDENTIFIER)[0];
                System.out.println(hits.score(j) + ": \t" + fileName);
            }
//        FileUtils.saveImageResultsToHtml("GeneralTest_testSearchIndexLarge_", hits, reader.document(10).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]);
            FileUtils.saveImageResultsToPng("GeneralTest_testSearchIndexLarge_" + i + "_", hits, reader.document(queryDocID).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]);
        }
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testRerankFilters() throws IOException {
    int queryDocID = (int) (Math.random() * 10000);
    IndexReader reader = DirectoryReader.open(FSDirectory.open(new File("index-large")));
    // select one feature for the large index:
    int featureIndex = 4;
    int count = 0;
    long ms = System.currentTimeMillis();
    ImageSearchHits hits = searchers[featureIndex].search(reader.document(queryDocID), reader);
    RerankFilter rerank = new RerankFilter(featureClasses[0], DocumentBuilder.FIELD_NAME_CEDD);
    LsaFilter lsa = new LsaFilter(featureClasses[0], DocumentBuilder.FIELD_NAME_CEDD);
    FileUtils.saveImageResultsToPng("GeneralTest_rerank_0_old", hits, reader.document(queryDocID).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]);
    hits = rerank.filter(hits, reader.document(queryDocID));
    FileUtils.saveImageResultsToPng("GeneralTest_rerank_1_new", hits, reader.document(queryDocID).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]);
    hits = lsa.filter(hits, reader.document(queryDocID));
    FileUtils.saveImageResultsToPng("GeneralTest_rerank_2_lsa", hits, reader.document(queryDocID).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]);
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testReUse() throws IOException, IllegalAccessException, InstantiationException {
        ArrayList<String> testFiles = FileUtils.getAllImages(new File("testdata/ferrari"), true);
        for (Class c : featureClasses) {
            LireFeature f1 = (LireFeature) c.newInstance();
            System.out.println(c.getName());
            for (String testFile : testFiles) {
                f1.extract(ImageIO.read(new File(testFile)));
                LireFeature f2 = (LireFeature) c.newInstance();
                f2.extract(ImageIO.read(new File(testFile)));
//                System.out.println(Arrays.toString(f1.getDoubleHistogram()));
//                System.out.println(Arrays.toString(f2.getDoubleHistogram()));
                assertEquals(f2.getDistance(f1), 0d, 0.000000001);
                f2.setByteArrayRepresentation(f1.getByteArrayRepresentation());
                assertEquals(f2.getDistance(f1), 0d, 0.000000001);
                byte[] tmp = new byte[1024*100];
                Arrays.fill(tmp, (byte) 0x000F);
                byte[] bytes = f1.getByteArrayRepresentation();
                System.arraycopy(bytes, 0, tmp, 12, bytes.length);
                f2.setByteArrayRepresentation(tmp, 12, bytes.length);
                assertEquals(f2.getDistance(f1), 0d, 0.000000001);
            }
        }
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testIndexSurfHistogram() throws IOException {
        // index all files
        System.out.println("-< Getting files to index >--------------");
        List<String> images = FileUtils.getAllImages(new File(directory), true);
        System.out.println("-< Indexing " + images.size() + " files >--------------");
        indexFiles(images, index);

        // create histograms
//        System.out.println("-< Creating SIFT based histograms >--------------");
//        SiftFeatureHistogramBuilder siftFeatureHistogramBuilder = new SiftFeatureHistogramBuilder(IndexReader.open(FSDirectory.open(new File(index))), numSamples, clusters);
//        siftFeatureHistogramBuilder.index();
        System.out.println("-< Creating SURF based histograms >--------------");
        SurfFeatureHistogramBuilder surfFeatureHistogramBuilder = new SurfFeatureHistogramBuilder(DirectoryReader.open(FSDirectory.open(new File(index))), numSamples, clusters);
        surfFeatureHistogramBuilder.index();
//        System.out.println("-< Creating MSER based histograms >--------------");
//        MSERFeatureHistogramBuilder mserFeatureHistogramBuilder = new MSERFeatureHistogramBuilder(IndexReader.open(FSDirectory.open(new File(index))), numSamples, clusters);
//        mserFeatureHistogramBuilder.index();
//        System.out.println("-< Finished >--------------");

    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testIndexingAndSearchSift() throws IOException {
    // Creating an Lucene IndexWriter
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40));
    IndexWriter iw = new IndexWriter(FSDirectory.open(indexPath), conf);
    long ms = System.currentTimeMillis();
    int count = 0;
    ArrayList<File> files = FileUtils.getAllImageFiles(new File("testdata\\ferrari"), true);
    for (Iterator<File> i = files.iterator(); i.hasNext(); ) {
        File imgFile = i.next();
        iw.addDocument(siftBuilder.createDocument(
                ImageIO.read(imgFile), imgFile.getPath()));
        count++;
        if (count > 100 && count % 500 == 0) {
            System.out.println(count + " files indexed. " + (System.currentTimeMillis() - ms) / (count) + " ms per file");
        }

    }
    iw.close();
    IndexReader ir = DirectoryReader.open(FSDirectory.open(indexPath));
    SiftFeatureHistogramBuilder sfh = new SiftFeatureHistogramBuilder(ir, 1000, 500);
    sfh.index();
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testCreateCEDDIndex() throws IOException {
    ArrayList<String> images = FileUtils.getAllImages(new File("E:/temp/flickrphotos/flickrphotos"), true);

    ChainedDocumentBuilder builder = new ChainedDocumentBuilder();
    builder.addBuilder(DocumentBuilderFactory.getCEDDDocumentBuilder());
    IndexWriter iw = LuceneUtils.createIndexWriter(indexPath + "-cedd", true);
    int count = 0;
    long ms = System.currentTimeMillis();
    for (String identifier : images) {
        try {
            Document doc = builder.createDocument(new FileInputStream(identifier), identifier);
            iw.addDocument(doc);
        } catch (Exception e) {
            System.err.print("\n ;-( ");//e.printStackTrace();
        }
        count++;
        if (count % 1000 == 0) System.out.println((100 * count) / images.size() + "% ");
    }
    System.out.println("Time taken: " + ((System.currentTimeMillis() - ms) / testFiles.length) + " ms");
    iw.commit();
    iw.close();
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testExtendedIndexMSER() throws IOException {
    MSERDocumentBuilder builder = new MSERDocumentBuilder();
    IndexWriterConfig conf = new IndexWriterConfig(LuceneUtils.LUCENE_VERSION,
            new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
    IndexWriter iw = new IndexWriter(FSDirectory.open(indexPath), conf);
    long ms = System.currentTimeMillis();
    int count = 0;
    ArrayList<File> files = FileUtils.getAllImageFiles(new File("D:\\DataSets\\WIPO\\CA\\sample"), true);
    for (Iterator<File> i = files.iterator(); i.hasNext(); ) {
        File imgFile = i.next();
        BufferedImage img = ImageIO.read(imgFile);
        if (Math.max(img.getWidth(), img.getHeight()) < 800) {
            // scale image ...
            img = ImageUtils.scaleImage(img, 800);
        }
        iw.addDocument(builder.createDocument(img, imgFile.getPath()));
        count++;
        if (count > 2 && count % 25 == 0) {
            System.out.println(count + " files indexed. " + (System.currentTimeMillis() - ms) / (count) + " ms per file");
        }

    }
    iw.close();
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testSerialization() throws IOException {
        int bytes = 0;
        int sum = 0;
        ArrayList<File> files = FileUtils.getAllImageFiles(new File("testdata/ferrari"), true);
        for (Iterator<File> iterator = files.iterator(); iterator.hasNext(); ) {
            File next = iterator.next();
            BufferedImage image = ImageIO.read(next);
            AutoColorCorrelogram f1 = new AutoColorCorrelogram();
            AutoColorCorrelogram f2 = new AutoColorCorrelogram();

            f1.extract(image);
            System.out.println("f1.getDoubleHistogram().length = " + f1.getDoubleHistogram().length);
            System.out.println(Arrays.toString(f1.getDoubleHistogram()));
            f2.setByteArrayRepresentation(f1.getByteArrayRepresentation());
//            System.out.println(Arrays.toString(f2.getDoubleHistogram()));
            assertTrue(f2.getDistance(f1) == 0);
//            boolean isSame = true;
//            for (int i = 0; i < f2..length; i++) {
//                if (f1.data[i] != f2.data[i]) isSame=false;
//            }
//            assertTrue(isSame);
        }
        double save = 1d - (double) bytes / (double) sum;
        System.out.println(save * 100 + "% saved");
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testExtraction() throws IOException {
    ArrayList<File> files = FileUtils.getAllImageFiles(new File("testdata/ferrari"), true);
    for (Iterator<File> iterator = files.iterator(); iterator.hasNext(); ) {
        File next = iterator.next();
        BufferedImage image = ImageIO.read(next);
        JCD f1 = new JCD();
        JCD f2 = new JCD();

        f1.extract(image);
        System.out.println(Arrays.toString(f1.getByteArrayRepresentation()));

        bytes += f1.getByteArrayRepresentation().length;
        sum += 168;
        f2.setByteArrayRepresentation(f1.getByteArrayRepresentation());
        assertTrue(f2.getDistance(f1) == 0);
    }
    double save = 1d - (double) bytes / (double) sum;
    System.out.println(save * 100 + "% saved");


}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testSiftIndexing() throws IOException {
        ArrayList<String> images = FileUtils.getAllImages(new File(testExtensive), true);
        ChainedDocumentBuilder db = new ChainedDocumentBuilder();
        db.addBuilder(new SiftDocumentBuilder());
        db.addBuilder(DocumentBuilderFactory.getCEDDDocumentBuilder());
        IndexWriter iw = LuceneUtils.createIndexWriter("sift-idx", true);

        for (int i = 0; i < images.size(); i++) {
//            int sampleQuery = sampleQueries[i];
//            String s = testExtensive + "/" + sampleQuery + ".jpg";
            iw.addDocument(db.createDocument(new FileInputStream(images.get(i)), images.get(i)));
            if (i % 100 == 99) System.out.print(".");
            if (i % 1000 == 999) System.out.print(" ~ " + i + " files indexed\n");
            if (i > 1000) break;
        }
        System.out.println("");
        iw.close();
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testSurfIndexing() throws IOException {
        ArrayList<String> images = FileUtils.getAllImages(new File(testExtensive), true);
        ChainedDocumentBuilder db = new ChainedDocumentBuilder();
        db.addBuilder(new SurfDocumentBuilder());
        IndexWriter iw = LuceneUtils.createIndexWriter("sift-idx", true);
        for (int i = 0; i < images.size(); i++) {
//            int sampleQuery = sampleQueries[i];
//            String s = testExtensive + "/" + sampleQuery + ".jpg";
            iw.addDocument(db.createDocument(new FileInputStream(images.get(i)), images.get(i)));
            if (i % 100 == 99) System.out.print(".");
            if (i % 1000 == 999) System.out.print(" ~ " + i + " files indexed\n");
            if (i > 1000) break;
        }
        System.out.println("");
        iw.close();
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testRerankFilter() throws IOException {
    // index images
    // indexFiles();
    // search
    System.out.println("---< searching >-------------------------");
    IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
    Document document = reader.document(0);
    ImageSearcher searcher = ImageSearcherFactory.createCEDDImageSearcher(100);
    ImageSearchHits hits = searcher.search(document, reader);
    // rerank
    System.out.println("---< filtering >-------------------------");
    RerankFilter filter = new RerankFilter(ColorLayout.class, DocumentBuilder.FIELD_NAME_COLORLAYOUT);
    hits = filter.filter(hits, document);

    // output
    FileUtils.saveImageResultsToHtml("filtertest", hits, document.getField(DocumentBuilder.FIELD_NAME_IDENTIFIER).stringValue());
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testLsaFilter() throws IOException {
        // index images
//        indexFiles();
        // search
        System.out.println("---< searching >-------------------------");
        IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
        Document document = reader.document(0);
        ImageSearcher searcher = ImageSearcherFactory.createCEDDImageSearcher(100);
        ImageSearchHits hits = searcher.search(document, reader);
        // rerank
        System.out.println("---< filtering >-------------------------");
        LsaFilter filter = new LsaFilter(CEDD.class, DocumentBuilder.FIELD_NAME_CEDD);
        hits = filter.filter(hits, document);

        // output
        FileUtils.saveImageResultsToHtml("filtertest", hits, document.getField(DocumentBuilder.FIELD_NAME_IDENTIFIER).stringValue());
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
@SuppressWarnings("unused")
private Document indexFiles() throws IOException {
       System.out.println("---< indexing >-------------------------");
       int count = 0;
       DocumentBuilder builder = getDocumentBuilder();
       ArrayList<String> allImages = FileUtils.getAllImages(new File("wang-1000"), true);
       IndexWriter iw = LuceneUtils.createIndexWriter(indexPath, true);
       Document document = null;
       for (Iterator<String> iterator = allImages.iterator(); iterator.hasNext(); ) {
           String filename = iterator.next();
           BufferedImage image = ImageIO.read(new FileInputStream(filename));
           document = builder.createDocument(image, filename);
           iw.addDocument(document);
           count++;
           if (count % 50 == 0)
               System.out.println("finished " + (count * 100) / allImages.size() + "% of the images.");
       }
       iw.close();
       return document;
   }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testIndexing() throws IOException {
        List<String> allImages = FileUtils.getAllImages(new File(filePath), true);
        System.out.println("Found " + allImages.size() + " files.");
        IndexWriter iw = LuceneUtils.createIndexWriter(indexPath, true);
//        ParallelIndexer pix = new ParallelIndexer(allImages, new MirFlickrDocumentBuilder());
        ParallelIndexer pix = new ParallelIndexer(allImages, new MetadataBuilder());
        new Thread(pix).start();
        Document doc;
        javax.swing.ProgressMonitor pm = new javax.swing.ProgressMonitor(null, "Indexing ...", "", 0, allImages.size());
        int count = 0;
        while ((doc = pix.getNext()) != null) {
            iw.addDocument(doc);
            pm.setProgress(++count);
            pm.setNote(count + " documents finished");
        }
        iw.close();
    }
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
    ArrayList<File> files = FileUtils.getAllImageFiles(new File("D:\\Temp\\tmp"), false);
    int count = 10;
    for (Iterator<File> iterator = files.iterator(); iterator.hasNext(); ) {
        File next = iterator.next();
        BufferedImage img = ImageIO.read(next);
        BufferedImage toWrite = new BufferedImage(img.getWidth()*2, img.getHeight(), BufferedImage.TYPE_INT_RGB);
        toWrite.getGraphics().drawImage(img, 0, 0, null);
        BufferedImage bufferedImage = clusterPixels(img);
        toWrite.getGraphics().drawImage(bufferedImage, img.getWidth(), 0, null);
        ImageIO.write(toWrite, "png", new File("out_test_"+count+".png"));
        count++;
    }
}
 

import net.semanticmetadata.lire.utils.FileUtils; //导入依赖的package包/类
public void testIndex() throws IOException {
    System.out.println("-< Getting files to index >--------------");
    ArrayList<String> images = FileUtils.getAllImages(new File(testFiles), true);
    System.out.println("-< Indexing " + images.size() + " files >--------------");

    indexFiles(images, builder, testIndex);
    System.out.println("-< Indexing finished >--------------");

}