// Handle #pragma
int TPpContext::CPPpragma(TPpToken* ppToken)
{
    char SrcStrName[2];
    TVector<TString> tokens;

    TSourceLoc loc = ppToken->loc;  // because we go to the next line before processing
    int token = scanToken(ppToken);
    while (token != '\n' && token != EndOfInput) {
        switch (token) {
        case PpAtomIdentifier:
        case PpAtomConstInt:
        case PpAtomConstUint:
        case PpAtomConstFloat:
        case PpAtomConstDouble:
            tokens.push_back(ppToken->name);
            break;
        default:
            SrcStrName[0] = (char)token;
            SrcStrName[1] = '\0';
            tokens.push_back(SrcStrName);
        }
        token = scanToken(ppToken);
    }

    if (token == EndOfInput)
        parseContext.ppError(loc, "directive must end with a newline", "#pragma", "");
    else
        parseContext.handlePragma(loc, tokens);

    return token;    
}

// Handle #error
int TPpContext::CPPerror(TPpToken* ppToken) 
{
    int token = scanToken(ppToken);
    std::string message;
    TSourceLoc loc = ppToken->loc;

    while (token != '\n' && token != EndOfInput) {
        if (token == PpAtomConstInt   || token == PpAtomConstUint   ||
            token == PpAtomConstInt64 || token == PpAtomConstUint64 ||
#ifdef AMD_EXTENSIONS
            token == PpAtomConstFloat16 ||
#endif
            token == PpAtomConstFloat || token == PpAtomConstDouble) {
                message.append(ppToken->name);
        } else if (token == PpAtomIdentifier || token == PpAtomConstString) {
            message.append(ppToken->name);
        } else {
            message.append(GetAtomString(token));
        }
        message.append(" ");
        token = scanToken(ppToken);
    }
    parseContext.notifyErrorDirective(loc.line, message.c_str());
    //store this msg into the shader's information log..set the Compile Error flag!!!!
    parseContext.ppError(loc, message.c_str(), "#error", "");

    return '\n';
}

// Handle #ifdef
int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
{
    int token = scanToken(ppToken);
    if (++ifdepth > maxIfNesting) {
        parseContext.ppError(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", "");
        return 0;
    }
    elsetracker++;
    if (token != PpAtomIdentifier) {
        if (defined)
            parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", "");
        else
            parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifndef", "");
    } else {
        MacroSymbol* macro = lookupMacroDef(atomStrings.getAtom(ppToken->name));
        token = scanToken(ppToken);
        if (token != '\n') {
            parseContext.ppError(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", "");
            while (token != '\n' && token != EndOfInput)
                token = scanToken(ppToken);
        }
        if (((macro != nullptr && !macro->undef) ? 1 : 0) != defined)
            token = CPPelse(1, ppToken);
    }

    return token;
}

// Expand macros, skipping empty expansions, to get to the first real token in those expansions.
int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
    while (token == PpAtomIdentifier && ppToken->atom != PpAtomDefined) {
        int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false);
        if (macroReturn == 0) {
            parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
            err = true;
            res = 0;
            token = scanToken(ppToken);
            break;
        }
        if (macroReturn == -1) {
            if (! shortCircuit && parseContext.profile == EEsProfile) {
                const char* message = "undefined macro in expression not allowed in es profile";
                if (parseContext.relaxedErrors())
                    parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
                else
                    parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
            }
        }
        token = scanToken(ppToken);
    }

    return token;
}

TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
{
    int token;
    TokenStream *n;
    RewindTokenStream(a);
    do {
        token = ReadToken(a, ppToken);
        if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
            break;
    } while (token != EndOfInput);

    if (token == EndOfInput)
        return a;

    n = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(a);
    while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
        if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
            continue;
        RecordToken(n, token, ppToken);
    }
    popInput();
    delete a;

    return n;
}

// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
    // expand the argument
    TokenStream* expandedArg = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(arg);
    int token;
    while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
        token = tokenPaste(token, *ppToken);
        if (token == tMarkerInput::marker || token == EndOfInput)
            break;
        if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0)
            continue;
        expandedArg->putToken(token, ppToken);
    }

    if (token == EndOfInput) {
        // MacroExpand ate the marker, so had bad input, recover
        delete expandedArg;
        expandedArg = nullptr;
    } else {
        // remove the marker
        popInput();
    }

    return expandedArg;
}

// Call when there should be no more tokens left on a line.
int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
{
    if (token != '\n' && token != EndOfInput) {
        static const char* message = "unexpected tokens following directive";

        const char* label;
        if (atom == PpAtomElse)
            label = "#else";
        else if (atom == PpAtomElif)
            label = "#elif";
        else if (atom == PpAtomEndif)
            label = "#endif";
        else if (atom == PpAtomIf)
            label = "#if";
        else if (atom == PpAtomLine)
            label = "#line";
        else
            label = "";

        if (parseContext.relaxedErrors())
            parseContext.ppWarn(ppToken->loc, message, label, "");
        else
            parseContext.ppError(ppToken->loc, message, label, "");

        while (token != '\n' && token != EndOfInput)
            token = scanToken(ppToken);
    }

    return token;
}

// Expand macros, skipping empty expansions, to get to the first real token in those expansions.
int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
    while (token == PpAtomIdentifier && strcmp("defined", ppToken->name) != 0) {
        switch (MacroExpand(ppToken, true, false)) {
        case MacroExpandNotStarted:
        case MacroExpandError:
            parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
            err = true;
            res = 0;
            break;
        case MacroExpandStarted:
            break;
        case MacroExpandUndef:
            if (! shortCircuit && parseContext.profile == EEsProfile) {
                const char* message = "undefined macro in expression not allowed in es profile";
                if (parseContext.relaxedErrors())
                    parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
                else
                    parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
            }
            break;
        }
        token = scanToken(ppToken);
        if (err)
            break;
    }

    return token;
}

//
// The main functional entry-point into the preprocessor, which will
// scan the source strings to figure out and return the next processing token.
//
// Return string pointer to next token.
// Return 0 when no more tokens.
//
const char* TPpContext::tokenize(TPpToken* ppToken)
{
    int token = '\n';

    for(;;) {
        const char* tokenString = nullptr;
        token = scanToken(ppToken);
        ppToken->token = token;
        if (token == EOF) {
            missingEndifCheck();
            return nullptr;
        }
        if (token == '#') {
            if (previous_token == '\n') {
                token = readCPPline(ppToken);
                if (token == EOF) {
                    missingEndifCheck();
                    return nullptr;
                }
                continue;
            } else {
                parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", "");
                return nullptr;
            }
        }
        previous_token = token;

        if (token == '\n')
            continue;

        // expand macros
        if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
            continue;

        if (token == CPP_IDENTIFIER)
            tokenString = GetAtomString(ppToken->atom);
        else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
                 token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
            tokenString = ppToken->name;
        else if (token == CPP_STRCONSTANT) {
            parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
            tokenString = nullptr;
        } else if (token == '\'') {
            parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
            tokenString = nullptr;
        } else
            tokenString = GetAtomString(token);

        if (tokenString) {
            if (tokenString[0] != 0)
                parseContext.tokensBeforeEOF = 1;

            return tokenString;
        }
    }
}

std::vector<Token> Scanner::scanTokens()
{
	while (!isAtEnd())
	{
		start_ = current_;
		scanToken();
	}
	tokens_.push_back(Token(EOF_T, "", "", line_));
	return tokens_;
}

// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
    // expand the argument
    TokenStream* expandedArg = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(arg);
    int token;
    while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
        token = tokenPaste(token, *ppToken);
        if (token == PpAtomIdentifier) {
            switch (MacroExpand(ppToken, false, newLineOkay)) {
            case MacroExpandNotStarted:
                break;
            case MacroExpandError:
                // toss the rest of the pushed-input argument by scanning until tMarkerInput
                while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput)
                    ;
                break;
            case MacroExpandStarted:
            case MacroExpandUndef:
                continue;
            }
        }
        if (token == tMarkerInput::marker || token == EndOfInput)
            break;
        expandedArg->putToken(token, ppToken);
    }

    if (token != tMarkerInput::marker) {
        // Error, or MacroExpand ate the marker, so had bad input, recover
        delete expandedArg;
        expandedArg = nullptr;
    }

    return expandedArg;
}

// fetchToken() will return the first token in the lookahead store (if the
// lookahead store has tokens) or it will get a new token from the frontier
int Scanner::fetchToken(ScannerToken &t, Location &l) {
  m_token = &t;
  m_loc = &l;
  int tokid;
  if (!m_lookahead.empty()) {
    // If there is a lookahead token, return that. No need to perform
    // special logic for "ReturnAllTokens", we already accounted for
    // that when the tokens were inserted into m_lookahead
    TokenStore::iterator it = m_lookahead.begin();
    tokid = it->t;
    *m_token = it->token;
    *m_loc = it->loc;
    return tokid;
  }
  return scanToken(t,l);
}

void scanLine(int line_no, char *line, FILE *result) {

	strtrim(line);
	if (strlen(line) == 0) return;
	
	int token_starts[128] = {0};
	int token_count = getTokenCountByStartIndexs(line, token_starts);

	int i;
	char token[32];
	int start, len;
	for (i = 0; i < token_count; i++) {

		start = token_starts[i];
		if (line[start] == ' ') continue;

		len = token_starts[i + 1] - start;
		substr(line, token, start, len);

		scanToken(token, line_no);
	}
}

 void Scanner::scan (Parser& parser)
 {
     while (scanToken (parser));
 }

//
// Do all token-pasting related combining of two pasted tokens when getting a
// stream of tokens from a replacement list. Degenerates to no processing if a
// replacement list is not the source of the token stream.
//
int TPpContext::tokenPaste(int token, TPpToken& ppToken)
{
    // starting with ## is illegal, skip to next token
    if (token == PpAtomPaste) {
        _parseContext.ppError(ppToken.loc, "unexpected location", "##", "");
        return scanToken(&ppToken);
    }

    int resultToken = token; // "foo" pasted with "35" is an identifier, not a number

    // ## can be chained, process all in the chain at once
    while (peekPasting()) {
        TPpToken pastedPpToken;

        // next token has to be ##
        token = scanToken(&pastedPpToken);
        assert(token == PpAtomPaste);

        // This covers end of macro expansion
        if (endOfReplacementList()) {
            _parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", "");
            break;
        }

        // get the token after the ##
        token = scanToken(&pastedPpToken);

        // This covers end of argument expansion
        if (token == tMarkerInput::marker) {
            _parseContext.ppError(ppToken.loc, "unexpected location; end of argument", "##", "");
            break;
        }

        // get the token text
        switch (resultToken) {
        case PpAtomIdentifier:
            // already have the correct text in token.names
            break;
        case '=':
        case '!':
        case '-':
        case '~':
        case '+':
        case '*':
        case '/':
        case '%':
        case '<':
        case '>':
        case '|':
        case '^':
        case '&':
        case PpAtomRight:
        case PpAtomLeft:
        case PpAtomAnd:
        case PpAtomOr:
        case PpAtomXor:
            strcpy(ppToken.name, atomStrings.getString(resultToken));
            strcpy(pastedPpToken.name, atomStrings.getString(token));
            break;
        default:
            _parseContext.ppError(ppToken.loc, "not supported for these tokens", "##", "");
            return resultToken;
        }

        // combine the tokens
        if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength) {
            _parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", "");
            return resultToken;
        }
        strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name));

        // correct the kind of token we are making, if needed (identifiers stay identifiers)
        if (resultToken != PpAtomIdentifier) {
            int newToken = atomStrings.getAtom(ppToken.name);
            if (newToken > 0)
                resultToken = newToken;
            else
                _parseContext.ppError(ppToken.loc, "combined token is invalid", "##", "");
        }
    }

    return resultToken;
}

//
// Check an identifier (atom) to see if it is a macro that should be expanded.
// If it is, and defined, push a tInput that will produce the appropriate expansion
// and return 1.
// If it is, but undefined, and expandUndef is requested, push a tInput that will 
// expand to 0 and return -1.
// Otherwise, return 0 to indicate no expansion, which is not necessarily an error.
//
int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay)
{
    ppToken->space = false;
    switch (atom) {
    case PpAtomLineMacro:
        ppToken->ival = parseContext.getCurrentLoc().line;
        sprintf(ppToken->name, "%d", ppToken->ival);
        UngetToken(PpAtomConstInt, ppToken);
        return 1;

    case PpAtomFileMacro: {
        if (parseContext.getCurrentLoc().name)
            parseContext.ppRequireExtensions(ppToken->loc, 1, &E_GL_GOOGLE_cpp_style_line_directive, "filename-based __FILE__");
        ppToken->ival = parseContext.getCurrentLoc().string;
        sprintf(ppToken->name, "%s", ppToken->loc.getStringNameOrNum().c_str());
        UngetToken(PpAtomConstInt, ppToken);
        return 1;
    }

    case PpAtomVersionMacro:
        ppToken->ival = parseContext.version;
        sprintf(ppToken->name, "%d", ppToken->ival);
        UngetToken(PpAtomConstInt, ppToken);
        return 1;

    default:
        break;
    }

    Symbol *sym = LookUpSymbol(atom);
    int token;
    int depth = 0;

    // no recursive expansions
    if (sym && sym->mac.busy)
        return 0;

    // not expanding undefined macros
    if ((! sym || sym->mac.undef) && ! expandUndef)
        return 0;

    // 0 is the value of an undefined macro
    if ((! sym || sym->mac.undef) && expandUndef) {
        pushInput(new tZeroInput(this));
        return -1;
    }

    tMacroInput *in = new tMacroInput(this);

    TSourceLoc loc = ppToken->loc;  // in case we go to the next line before discovering the error
    in->mac = &sym->mac;
    if (sym->mac.args) {
        token = scanToken(ppToken);
        if (newLineOkay) {
            while (token == '\n')                
                token = scanToken(ppToken);
        }
        if (token != '(') {
            parseContext.ppError(loc, "expected '(' following", "macro expansion", GetAtomString(atom));
            UngetToken(token, ppToken);
            ppToken->atom = atom;

            delete in;
            return 0;
        }
        in->args.resize(in->mac->argc);
        for (int i = 0; i < in->mac->argc; i++)
            in->args[i] = new TokenStream;
        int arg = 0;
        bool tokenRecorded = false;
        do {
            depth = 0;
            while (1) {
                token = scanToken(ppToken);
                if (token == EndOfInput) {
                    parseContext.ppError(loc, "End of input in macro", "macro expansion", GetAtomString(atom));
                    delete in;
                    return 0;
                }
                if (token == '\n') {
                    if (! newLineOkay) {
                        parseContext.ppError(loc, "End of line in macro substitution:", "macro expansion", GetAtomString(atom));
                        delete in;
                        return 0;
                    }
                    continue;
                }
                if (token == '#') {
                    parseContext.ppError(ppToken->loc, "unexpected '#'", "macro expansion", GetAtomString(atom));
                    delete in;
                    return 0;
                }
//.........这里部分代码省略.........