import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private List<Comparator> loadDefaultComparators() {
 String PKG = "no.priv.garshol.duke.comparators.";
 String[] compnames = new String[] {
	  "DiceCoefficientComparator",
	  "DifferentComparator",
	  "ExactComparator",
	  "JaroWinkler",
	  "JaroWinklerTokenized",
	  "Levenshtein",
	  "NumericComparator",
	  "PersonNameComparator",
	  "SoundexComparator",
	  "WeightedLevenshtein",
	  "NorphoneComparator",
	  "MetaphoneComparator",
	  "QGramComparator",
	  "GeopositionComparator",
	  "LongestCommonSubstring",
 };

 List<Comparator> comparators = new ArrayList<Comparator>();
 for (int ix = 0; ix < compnames.length; ix++)
  comparators.add((Comparator)ObjectUtils.instantiate(PKG + compnames[ix]));
 
 return comparators;  
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private static Comparator makeComparator(String measure) {
  switch (measure) {
    case "jaccard":
    case "JaccardIndex":
      return new JaccardIndexComparator();
    case "jw":
    case "JaroWinkler":
      // JaroWinkler Comparator contains bug which will be fixed in Duke 1.3 release.
      // Before that happens, we duplicate the JaroWinkler comparator class with the bug fixed
      return new H2OJaroWinklerComparator();
    case "lv":
    case "Levenshtein":
      return new LevenshteinDistanceComparator();
    case "lcs":
    case "LongestCommonSubstring":
      return new LongestCommonSubstring();
    case "qgram":
    case "QGram":
      return new QGramComparator();
    case "soundex":
    case "Soundex":
      return new SoundexComparator();
    default:
      throw new IllegalArgumentException("Unknown comparator: " + measure);
  }
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private static double computeProb(Collection<String> vs1, Collection<String> vs2, Comparator comp, List<Cleaner> cleanersList, Double max, Double min) {
    double high = 0.0;
    for (String v1 : vs1) {
        if (v1.equals("")) {
            continue;
        }

        v2fieldloop:
        for (String v2 : vs2) {
            if (v2.equals("")) {
                continue;
            }

            for (Cleaner cl : cleanersList) {
                v2 = cl.clean(v2);
                if ((v2 == null) || v2.equals("")) {
                    continue v2fieldloop;
                }
            }
            double p = compare(v1, v2, max, min, comp);
            high = Math.max(high, p);
        }
    }
    return high;
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
/**
 * Returns the probability that the records v1 and v2 came from represent
 * the same entity, based on high and low probability settings etc.
 *
 * @param v1         1st String
 * @param v2         2nd String
 * @param high       max probability
 * @param low        min probability
 * @param comparator the comparator to use
 * @return the computed probability
 */
private static double compare(
        final String v1,
        final String v2,
        final double high,
        final double low,
        final Comparator comparator) {

    if (comparator == null) {
        return AVERAGE_SCORE; // we ignore properties with no comparator
    }

    double sim = comparator.compare(v1, v2);
    if (sim < AVERAGE_SCORE) {
        return low;
    } else {
        return ((high - AVERAGE_SCORE) * (sim * sim)) + AVERAGE_SCORE;
    }
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
/**
  * Creates an initial copy of the starting configuration, with no
  * changes. Used to initialize the aspects list.
  * @param mutation_rate Mutation rate. -1 if self-evolving.
  * @param recombination_rate Recombination rate. -1.0 if self-evolving.
  * @param evolve_comparators If false the comparators will be left as
  *                           in the original configuration and not evolved.
  */
 public GeneticConfiguration(Configuration config, int mutation_rate,
                             double recombination_rate,
                             boolean evolve_comparators) {
  
   this.config = config;
List<Comparator> comparators = loadDefaultComparators();
comparators.addAll(config.getCustomComparators());
   
   this.aspects = new ArrayList();
   aspects.add(new ThresholdAspect());
   for (Property prop : config.getProperties()) {
     if (!prop.isIdProperty()) {
       if (evolve_comparators)
         aspects.add(new ComparatorAspect(prop, comparators));
       aspects.add(new LowProbabilityAspect(prop));
       aspects.add(new HighProbabilityAspect(prop));
     }
   }
   if (mutation_rate == -1)
     aspects.add(new MutationRateAspect());
   else
     this.mutation_rate = mutation_rate;
   if (recombination_rate == -1.0)
     aspects.add(new RecombinationRateAspect());
   else
     this.recombination_rate = recombination_rate;
 }
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private List<Comparator> loadDefaultComparators() {
 String PKG = "no.priv.garshol.duke.comparators.";
 String[] compnames = new String[] {
	  "DiceCoefficientComparator",
	  "DifferentComparator",
	  "ExactComparator",
	  "JaroWinkler",
	  "JaroWinklerTokenized",
	  "Levenshtein",
	  "NumericComparator",
	  "PersonNameComparator",
	  "SoundexComparator",
	  "WeightedLevenshtein",
	  "NorphoneComparator",
	  "MetaphoneComparator",
	  "QGramComparator",
	  "GeopositionComparator",
	  "LongestCommonSubstring",
	  "XSDDateTimeComparator"
 };

 List<Comparator> comparators = new ArrayList<Comparator>();
 for (int ix = 0; ix < compnames.length; ix++)
  comparators.add((Comparator)ObjectUtils.instantiate(PKG + compnames[ix]));
 
 return comparators;  
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
@Test
 public void canAddCustomComparator() {
   GeneticConfiguration conf = new GeneticConfiguration(config1);	    
Property aspectProp = new PropertyImpl(propName, null, 0.5, 0.5);
List<Comparator> compList = new ArrayList<Comparator>();
compList.add(comparator);
ComparatorAspect aspect = new ComparatorAspect(aspectProp, compList);

aspect.setRandomly(conf);

Property updatedProp = config1.getPropertyByName(propName);
Comparator randomComparator = updatedProp.getComparator();
   assertTrue("should have custom comparator set, but has : " + randomComparator.getClass(), randomComparator.equals(comparator));
 }
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
@Test
public void testCustomComparator() throws IOException, SAXException {
  Configuration config = ConfigLoader.load("classpath:config-custom-comparator.xml");

  List<Comparator> comparators = config.getCustomComparators();

  assertEquals(1, comparators.size());
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
@Test
public void testCustomEstimator() throws IOException, SAXException {
  Configuration config = ConfigLoader.load("classpath:config-custom-estimator.xml");

  List<Comparator> comparators = config.getCustomComparators();
  assertEquals(1, comparators.size());

  WeightedLevenshtein wl = (WeightedLevenshtein) comparators.get(0);
  DefaultWeightEstimator est = (DefaultWeightEstimator) wl.getEstimator();
  assertEquals(3.8, est.getDigitWeight());
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
/**
 * . Configures with data from JSON payload only
 *
 * @param fieldsParams fields parameters from JSON
 * @return the record for comparison
 */
private Record configureWithFieldsOnly(
        final List<Map<String, Object>> fieldsParams) {

    Map<String, Collection<String>> props =
            new HashMap<>();
    entityParams = new HashMap<>();

    for (Map<String, Object> value : fieldsParams) {
        HashMap<String, Object> map = new HashMap<>();

        String field = (String) value.get("field");

        List<Cleaner> cleanList =
                getCleaners((ArrayList<Map<String, String>>) value.get(CLEANERS));

        map.put(CLEANERS, cleanList);

        Double maxValue = 0.0;
        if (value.get(HIGH) != null) {
            maxValue = (Double) value.get(HIGH);
        }

        Double minValue = 0.0;
        if (value.get(LOW) != null) {
            minValue = (Double) value.get(LOW);
        }

        Comparator comp = getComparator(value);

        map.put(HIGH, maxValue);
        map.put(LOW, minValue);
        map.put(COMPARATOR, comp);

        entityParams.put(field, map);
    }

    readFields(fieldsParams, props);
    return new RecordImpl(props);
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private boolean isFuzzy(String fieldName) {
  Comparator c = config.getPropertyByName(fieldName).getComparator();
  return c != null && c.isTokenized();
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
public ComparatorAspect(Property prop, List<Comparator> comparators) {
  this.prop = prop;
  this.comparators = comparators;
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
private String shortname(Comparator comp) {
  return comp.getClass().getSimpleName();
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
public void setComparator(Comparator comp) {
  this.subcomp = comp;
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
@Test
public void testSmallData() throws IOException {
  File outfile = tmpdir.newFile("test.csv");

  FileWriter out = new FileWriter(outfile);
  out.write("id;name;age\n");
  out.write("1;LMG;39\n");
  out.write("2;GOG;40\n");
  out.write("3;GDM;29\n");
  out.write("4;AB;49\n");
  out.close();

  File tstfile = tmpdir.newFile("testfile.csv");
  out = new FileWriter(tstfile);
  out.close();

  CSVDataSource csv = new CSVDataSource();
  csv.setSeparator(';');
  csv.setInputFile(outfile.getAbsolutePath());
  csv.addColumn(new Column("id", null, null, null));
  csv.addColumn(new Column("name", null, null, null));
  csv.addColumn(new Column("age", null, null, null));

  ConfigurationImpl cfg = new ConfigurationImpl();
  cfg.addDatabase(new InMemoryDatabase());
  cfg.addDataSource(0, csv);

  Comparator cmp = new ExactComparator();

  List<Property> props = new ArrayList();
  props.add(new PropertyImpl("id"));
  props.add(new PropertyImpl("name", cmp, 0.0, 1.0));
  props.add(new PropertyImpl("age", cmp, 0.0, 1.0));

  cfg.setProperties(props);

  GeneticAlgorithm gen = new GeneticAlgorithm(cfg, 
                                              tstfile.getAbsolutePath(),
                                              true);
  gen.setQuiet(true);
  gen.run(); // should not crash!
}
 

import no.priv.garshol.duke.Comparator; //导入依赖的package包/类
@Test
public void testSmallData() throws IOException {
  File outfile = tmpdir.newFile("test.csv");

  FileWriter out = new FileWriter(outfile);
  out.write("id;name;age\n");
  out.write("1;LMG;39\n");
  out.write("2;GOG;40\n");
  out.write("3;GDM;29\n");
  out.write("4;AB;49\n");
  out.close();

  File tstfile = tmpdir.newFile("testfile.csv");
  out = new FileWriter(tstfile);
  out.close();

  CSVDataSource csv = new CSVDataSource();
  csv.setSeparator(';');
  csv.setInputFile(outfile.getAbsolutePath());
  csv.addColumn(new Column("id", null, null, null));
  csv.addColumn(new Column("name", null, null, null));
  csv.addColumn(new Column("age", null, null, null));

  ConfigurationImpl cfg = new ConfigurationImpl();
  cfg.addDatabase(new InMemoryDatabase());
  cfg.addDataSource(0, csv);

  Comparator cmp = new ExactComparator();

  List<Property> props = new ArrayList();
  props.add(new PropertyImpl("id"));
  props.add(new PropertyImpl("name", cmp, 0.0, 1.0));
  props.add(new PropertyImpl("age", cmp, 0.0, 1.0));

  cfg.setProperties(props);

  GeneticAlgorithm gen = new GeneticAlgorithm(cfg, tstfile.getAbsolutePath(),
                                              true);
  gen.setQuiet(true);
  gen.run(); // should not crash!
}