import edu.stanford.nlp.process.WhitespaceTokenizer; //导入依赖的package包/类
public WhitespaceTokenizerAnnotator(Properties props) {
  super(false);
  boolean eolIsSignificant =
    Boolean.valueOf(props.getProperty(EOL_PROPERTY, "false"));
  eolIsSignificant =
    (eolIsSignificant ||
     Boolean.valueOf(props.getProperty
                     (StanfordCoreNLP.NEWLINE_SPLITTER_PROPERTY, "false")));
  factory = new WhitespaceTokenizer.WhitespaceTokenizerFactory<CoreLabel>
            (new CoreLabelTokenFactory(), eolIsSignificant);
}
 

import edu.stanford.nlp.process.WhitespaceTokenizer; //导入依赖的package包/类
/**
 * Return a tokenizer which might be suitable for tokenizing text that
 * will be used with this Treebank/Language pair, without tokenizing carriage returns (i.e., treating them as white space).  The implementation in AbstractTreebankLanguagePack
 * returns a factory for {@link WhitespaceTokenizer}.
 *
 * @return A tokenizer
 */
public TokenizerFactory<? extends HasWord> getTokenizerFactory() {
  return WhitespaceTokenizer.factory(false);
}
 

import edu.stanford.nlp.process.WhitespaceTokenizer; //导入依赖的package包/类
/**
 * Return a tokenizer which might be suitable for tokenizing text that
 * will be used with this Treebank/Language pair, without tokenizing carriage returns (i.e., treating them as white space).  The implementation in AbstractTreebankLanguagePack
 * returns a factory for {@link WhitespaceTokenizer}.
 *
 * @return A tokenizer
 */
@Override
public TokenizerFactory<? extends HasWord> getTokenizerFactory() {
  return WhitespaceTokenizer.factory(false);
}