import edu.stanford.nlp.process.WordSegmentingTokenizer; //导入依赖的package包/类
/**
 * Tokenizes the highlighted text (using a tokenizer appropriate for the
 * selected language, and initiates the ParseThread to parse the tokenized
 * text.
 */
public void parse() {
  if (textPane.getText().length() == 0) {
    return;
  }

  // use endIndex+1 because substring subtracts 1
  String text = textPane.getText().substring(startIndex, endIndex + 1).trim();

  if (parser != null && text.length() > 0) {
    if (segmentWords) {
      ChineseLexiconAndWordSegmenter lex = (ChineseLexiconAndWordSegmenter) parser.getLexicon();
      ChineseTreebankLanguagePack.setTokenizerFactory(WordSegmentingTokenizer.factory(lex));
    }
    Tokenizer<? extends HasWord> toke = tlp.getTokenizerFactory().getTokenizer(new CharArrayReader(text.toCharArray()));
    List<? extends HasWord> wordList = toke.tokenize();
    parseThread = new ParseThread(wordList);
    parseThread.start();
    startProgressMonitor("Parsing", PARSE_TIME);
  }
}
 

import edu.stanford.nlp.process.WordSegmentingTokenizer; //导入依赖的package包/类
/**
 * Tokenizes the highlighted text (using a tokenizer appropriate for the
 * selected language, and initiates the ParseThread to parse the tokenized
 * text.
 */
public void parse() {
  if (textPane.getText().length() == 0) {
    return;
  }

  // use endIndex+1 because substring subtracts 1
  String text = textPane.getText().substring(startIndex, endIndex + 1).trim();

  if (parser != null && text.length() > 0) {
    if (segmentWords) {
      ChineseLexiconAndWordSegmenter lex = (ChineseLexiconAndWordSegmenter) parser.getLexicon();
      ChineseTreebankLanguagePack.setTokenizerFactory(WordSegmentingTokenizer.factory(lex));
    }
    //Tokenizer<? extends HasWord> toke = tlp.getTokenizerFactory().getTokenizer(new CharArrayReader(text.toCharArray()));
    Tokenizer<? extends HasWord> toke = tlp.getTokenizerFactory().getTokenizer(new StringReader(text));
    List<? extends HasWord> wordList = toke.tokenize();
    parseThread = new ParseThread(wordList);
    parseThread.start();
    startProgressMonitor("Parsing", PARSE_TIME);
  }
}
 

import edu.stanford.nlp.process.WordSegmentingTokenizer; //导入依赖的package包/类
/**
 * Returns a ChineseLexicon
 */
@Override
public Lexicon lex(Options op, Index<String> wordIndex, Index<String> tagIndex) {
  if (useCharacterBasedLexicon) {
    return lex = new ChineseCharacterBasedLexicon(this, wordIndex, tagIndex);
  // } else if (useMaxentLexicon) {
  // return lex = new ChineseMaxentLexicon();
  }
  if (op.lexOptions.uwModelTrainer == null) {
    op.lexOptions.uwModelTrainer = "edu.stanford.nlp.parser.lexparser.ChineseUnknownWordModelTrainer";
  }
  ChineseLexicon clex = new ChineseLexicon(op, this, wordIndex, tagIndex);
  if (segmenterClass != null) {
    try {
      segmenter = ReflectionLoading.loadByReflection(segmenterClass, this,
                                                     wordIndex, tagIndex);
    } catch (ReflectionLoading.ReflectionLoadingException e) {
      segmenter = ReflectionLoading.loadByReflection(segmenterClass);
    }
  }

  if (segmenter != null) {
    lex = new ChineseLexiconAndWordSegmenter(clex, segmenter);
    ctlp.setTokenizerFactory(WordSegmentingTokenizer.factory(segmenter));
  } else {
    lex = clex;
  }

  return lex;
}
 

import edu.stanford.nlp.process.WordSegmentingTokenizer; //导入依赖的package包/类
public ChineseLexiconAndWordSegmenter(ChineseLexicon lex, WordSegmenter seg) {
  chineseLexicon = lex;
  wordSegmenter = seg;
  ChineseTreebankLanguagePack.setTokenizerFactory(WordSegmentingTokenizer.factory(seg));
}
 

import edu.stanford.nlp.process.WordSegmentingTokenizer; //导入依赖的package包/类
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
  in.defaultReadObject();
  ChineseTreebankLanguagePack.setTokenizerFactory(WordSegmentingTokenizer.factory(wordSegmenter));
}