import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
@Override
public List<String> tokenize(String source) {
	try {
		char[] charArray = source.toCharArray();
           scanner.setSource(source.toCharArray());
           List<String> tokens = new ArrayList<>();
           int token;
           while ((token = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
               int tokenStart = scanner.getCurrentTokenStartPosition();
               int tokenEnd = scanner.getCurrentTokenEndPosition();
               if (token != ITerminalSymbols.TokenNameWHITESPACE) {
               	tokens.add(new String(charArray, tokenStart, tokenEnd - tokenStart + 1));
               }
           }
           return tokens;
       } catch (InvalidInputException e) {
           throw new RuntimeException(e);
       }
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * Method to check with methods share common attributes, according to
 * CK definition.
 * @author Mariana Azevedo
 * @since 13/07/2014
 * @param methods
 */
private void checkMethodsWithSharedAttributes(IMethod[] methods){
	
	IScanner scanner = null;
	for (IMethod method : methods) {
		String methodName = method.getElementName();
		
		try {
			scanner = ToolFactory.createScanner(false, false, false, false);
			scanner.setSource(method.getSource().toCharArray());
			while(true){
				int charactere = scanner.getNextToken();
				if (charactere == ITerminalSymbols.TokenNameEOF) break;
				if (charactere == ITerminalSymbols.TokenNameIdentifier) {
					addMethods(new String(scanner.getCurrentTokenSource()), methodName);
				}
			}
		} catch (JavaModelException exception1) {
			logger.error(exception1);
		} catch (InvalidInputException exception2) {
			logger.error(exception2);
		}
		
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * Removes comments and whitespace
 *
 * @param reference
 *            the type reference
 * @return the reference only consisting of dots and java identifier
 *         characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner = ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb = new StringBuffer();
	try {
		int tokenType = scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType = scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference = sb.toString();
	return reference;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
  if (start == end) return true;
  Assert.isTrue(start <= end);
  String trimmedText = buffer.getText(start, end - start).trim();
  if (0 == trimmedText.length()) {
    return true;
  } else {
    IScanner scanner = ToolFactory.createScanner(false, false, false, null);
    scanner.setSource(trimmedText.toCharArray());
    try {
      return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
    } catch (InvalidInputException e) {
      return false;
    }
  }
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private void doScan() {
  try {
    int token = fScanner.getNextToken();
    while (token != ITerminalSymbols.TokenNameEOF) {
      switch (token) {
        case ITerminalSymbols.TokenNameStringLiteral:
        case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
        case ITerminalSymbols.TokenNameCOMMENT_LINE:
        case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
          parseCurrentToken();
      }
      token = fScanner.getNextToken();
    }
  } catch (InvalidInputException e) {
    // ignore
  }
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * Removes comments and whitespace
 *
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
  IScanner scanner = ToolFactory.createScanner(false, false, false, false);
  scanner.setSource(reference.toCharArray());
  StringBuffer sb = new StringBuffer();
  try {
    int tokenType = scanner.getNextToken();
    while (tokenType != ITerminalSymbols.TokenNameEOF) {
      sb.append(scanner.getRawTokenSource());
      tokenType = scanner.getNextToken();
    }
  } catch (InvalidInputException e) {
    Assert.isTrue(false, reference);
  }
  reference = sb.toString();
  return reference;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private int getSurroundingComment(IScanner scanner) {
  try {
    int start = fLocation.getOffset();
    int end = start + fLocation.getLength();

    int token = scanner.getNextToken();
    while (token != ITerminalSymbols.TokenNameEOF) {
      if (TokenScanner.isComment(token)) {
        int currStart = scanner.getCurrentTokenStartPosition();
        int currEnd = scanner.getCurrentTokenEndPosition() + 1;
        if (currStart <= start && end <= currEnd) {
          return token;
        }
      }
      token = scanner.getNextToken();
    }

  } catch (InvalidInputException e) {
    // ignore
  }
  return ITerminalSymbols.TokenNameEOF;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * This method is used to retrieve the start position of the block.
 * @return int the dimension found, -1 if none
 */
protected int retrieveStartBlockPosition(int start, int end) {
	this.scanner.resetTo(start, end);
	try {
		int token;
		while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) {
			switch(token) {
				case TerminalTokens.TokenNameLBRACE://110
					return this.scanner.startPosition;
			}
		}
	} catch(InvalidInputException e) {
		// ignore
	}
	return -1;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private final void consumeDigits(int radix, boolean expectingDigitFirst) throws InvalidInputException {
	final int USING_UNDERSCORE = 1;
	final int INVALID_POSITION = 2;
	switch(consumeDigits0(radix, USING_UNDERSCORE, INVALID_POSITION, expectingDigitFirst)) {
		case USING_UNDERSCORE :
			if (this.sourceLevel < ClassFileConstants.JDK1_7) {
				throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17);
			}
			break;
		case INVALID_POSITION :
			if (this.sourceLevel < ClassFileConstants.JDK1_7) {
				throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17);
			}
			throw new InvalidInputException(INVALID_UNDERSCORE);
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * This method is used to retrieve the position after the right parenthesis.
 * @return int the position found
 */
protected int retrieveEndOfRightParenthesisPosition(int start, int end) {
	this.scanner.resetTo(start, end);
	try {
		int token;
		while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) {
			switch(token) {
				case TerminalTokens.TokenNameRPAREN:
					return this.scanner.currentPosition;
			}
		}
	} catch(InvalidInputException e) {
		// ignore
	}
	return -1;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end)
		return true;
	Assert.isTrue(start <= end);
	String trimmedText= buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner= ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
protected Object createFieldReference(Object receiver) throws InvalidInputException {
	try {
		MemberRef fieldRef = this.ast.newMemberRef();
		SimpleName fieldName = new SimpleName(this.ast);
		fieldName.internalSetIdentifier(new String(this.identifierStack[0]));
		fieldRef.setName(fieldName);
		int start = (int) (this.identifierPositionStack[0] >>> 32);
		int end = (int) this.identifierPositionStack[0];
		fieldName.setSourceRange(start, end - start + 1);
		if (receiver == null) {
			start = this.memberStart;
			fieldRef.setSourceRange(start, end - start + 1);
		} else {
			Name typeRef = (Name) receiver;
			fieldRef.setQualifier(typeRef);
			start = typeRef.getStartPosition();
			end = fieldName.getStartPosition()+fieldName.getLength()-1;
			fieldRef.setSourceRange(start, end-start+1);
		}
		return fieldRef;
	}
	catch (ClassCastException ex) {
			throw new InvalidInputException();
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
protected Object syntaxRecoverQualifiedName(int primitiveToken) throws InvalidInputException {
	if (this.cursorLocation == ((int)this.identifierPositionStack[this.identifierPtr])) {
		// special case of completion just before the dot.
		return createTypeReference(primitiveToken);
	}
	int idLength = this.identifierLengthStack[this.identifierLengthPtr];
	char[][] tokens = new char[idLength][];
	int startPtr = this.identifierPtr-idLength+1;
	System.arraycopy(this.identifierStack, startPtr, tokens, 0, idLength);
	long[] positions = new long[idLength+1];
	System.arraycopy(this.identifierPositionStack, startPtr, positions, 0, idLength);
	positions[idLength] = (((long)this.tokenPreviousPosition)<<32) + this.tokenPreviousPosition;
	this.completionNode = new CompletionOnJavadocQualifiedTypeReference(tokens, CharOperation.NO_CHAR, positions, this.tagSourceStart, this.tagSourceEnd);

	if (CompletionEngine.DEBUG) {
		System.out.println("	completion partial qualified type="+this.completionNode); //$NON-NLS-1$
	}
	return this.completionNode;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
/**
 * This method is used to retrieve position before the next right brace or semi-colon.
 * @return int the position found.
 */
protected int retrieveRightBrace(int start, int end) {
	this.scanner.resetTo(start, end);
	try {
		int token;
		while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) {
			switch(token) {
				case TerminalTokens.TokenNameRBRACE :
					return this.scanner.currentPosition - 1;
			}
		}
	} catch(InvalidInputException e) {
		// ignore
	}
	return -1;
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
protected Object createFieldReference(Object receiver) throws InvalidInputException {
	int refStart = (int) (this.identifierPositionStack[0] >>> 32);
	int refEnd = (int) this.identifierPositionStack[0];
	boolean inCompletion = (refStart <= (this.cursorLocation+1) && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers
		|| ((refStart == (refEnd+1) && refEnd == this.cursorLocation)) // or it's a completion on empty token
		|| (this.memberStart == this.cursorLocation); // or it's a completion just after the member separator with an identifier after the cursor
	if (inCompletion) {
		JavadocFieldReference fieldRef = (JavadocFieldReference) super.createFieldReference(receiver);
		char[] name = this.sourceParser.compilationUnit.getMainTypeName();
		TypeDeclaration typeDecl = getParsedTypeDeclaration();
		if (typeDecl != null) {
			name = typeDecl.name;
		}
		this.completionNode = new CompletionOnJavadocFieldReference(fieldRef, this.memberStart, name);
		if (CompletionEngine.DEBUG) {
			System.out.println("	completion field="+this.completionNode); //$NON-NLS-1$
		}
		return this.completionNode;
	}
	return super.createFieldReference(receiver);
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
public void printNextToken(int[] expectedTokenTypes, boolean considerSpaceIfAny){
	printComment(CodeFormatter.K_UNKNOWN, NO_TRAILING_COMMENT);
	try {
		this.currentToken = this.scanner.getNextToken();
		if (Arrays.binarySearch(expectedTokenTypes, this.currentToken) < 0) {
			StringBuffer expectations = new StringBuffer(5);
			for (int i = 0; i < expectedTokenTypes.length; i++){
				if (i > 0) {
					expectations.append(',');
				}
				expectations.append(expectedTokenTypes[i]);
			}
			throw new AbortFormatting("unexpected token type, expecting:["+expectations.toString()+"], actual:"+this.currentToken);//$NON-NLS-1$//$NON-NLS-2$
		}
		print(this.scanner.currentPosition - this.scanner.startPosition, considerSpaceIfAny);
	} catch (InvalidInputException e) {
		throw new AbortFormatting(e);
	}
}
 

import org.eclipse.jdt.core.compiler.InvalidInputException; //导入依赖的package包/类
private boolean isNextToken(int tokenName) {
	this.localScanner.resetTo(this.scribe.scanner.currentPosition, this.scribe.scannerEndPosition - 1);
	try {
		int token = this.localScanner.getNextToken();
		loop: while(true) {
			switch(token) {
				case TerminalTokens.TokenNameCOMMENT_BLOCK :
				case TerminalTokens.TokenNameCOMMENT_JAVADOC :
				case TerminalTokens.TokenNameCOMMENT_LINE :
					token = this.localScanner.getNextToken();
					continue loop;
				default:
					break loop;
			}
		}
		return  token == tokenName;
	} catch(InvalidInputException e) {
		// ignore
	}
	return false;
}