import net.sourceforge.pmd.cpd.SourceCode; //导入依赖的package包/类
/**
 * Builds and stores a {@link TokenEntry} from the source code.
 *
 * @param source source code.
 * @param cpdTokens tokens.
 * @throws IOException when it can't read a source file.
 */
@Override
public void tokenize(SourceCode source, Tokens cpdTokens) throws IOException {
    Lexer lexer = ApexLexer.create(config);
    String fileName = source.getFileName();
    List<Token> tokens = lexer.lex(new File(fileName));
    tokens.forEach(token -> {
        TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
        cpdTokens.add(cpdToken);
    });
    cpdTokens.add(TokenEntry.getEOF());
}
 

import net.sourceforge.pmd.cpd.SourceCode; //导入依赖的package包/类
@Override
public final void tokenize(SourceCode source, Tokens cpdTokens) {
  Lexer lexer = PuppetLexer.create(new PuppetConfiguration(charset));
  String fileName = source.getFileName();
  List<Token> tokens = lexer.lex(new File(fileName));
  for (Token token : tokens) {
    if (!token.getType().equals(PuppetTokenType.NEWLINE) && !token.getType().equals(PuppetTokenType.DEDENT) && !token.getType().equals(PuppetTokenType.INDENT)) {
      TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
      cpdTokens.add(cpdToken);
    }
  }
  cpdTokens.add(TokenEntry.getEOF());
}
 

import net.sourceforge.pmd.cpd.SourceCode; //导入依赖的package包/类
public final void tokenize(SourceCode source, Tokens cpdTokens) {
	Lexer lexer = PlSQLLexer.create(new PlSQLConfiguration(charset));
	String fileName = source.getFileName();
	List<Token> tokens = lexer.lex(new File(fileName));
	for (Token token : tokens) {
		TokenEntry cpdToken = new TokenEntry(getTokenImage(token),
				fileName, token.getLine());
		cpdTokens.add(cpdToken);
	}
	cpdTokens.add(TokenEntry.getEOF());
}