bool TokenName( unsigned token, char **start, unsigned *len )
{
    switch( token ) {
    case T_LINE_SEPARATOR:
        *start = LIT( End_Of_Line );
        *len = strlen( LIT( End_Of_Line ) ) + 1;
        return( TRUE );
    case T_INT_NUM:
    case T_REAL_NUM:
        *start = LIT( Num_Name );
        *len = strlen( LIT( Num_Name ) ) + 1;
        return( TRUE );
    case T_NAME:
        *start = LIT( Sym_Name_Name );
        *len = strlen( LIT( Sym_Name_Name ) ) + 1;
        return( TRUE );
    }
    if( token < LAST_CMDLN_DELIM ) {
        *start = &CmdLnDelimTab[ token - FIRST_CMDLN_DELIM ];
        *len = sizeof( char );
        return( TRUE );
    }
    if( ExprTokens != NULL ) {
        if( FindToken( ExprTokens->delims, token, start, len ) ) return(TRUE);
        if( FindToken( ExprTokens->keywords, token, start, len ) ) return(TRUE);
    }
    return( FALSE );
}

void Parser::SearchForEpisodeTitle() {
  auto token_begin = tokens_.begin();
  auto token_end = tokens_.begin();

  do {
    // Find the first non-enclosed unknown token
    token_begin = FindToken(token_end, tokens_.end(),
                            kFlagNotEnclosed | kFlagUnknown);
    if (token_begin == tokens_.end())
      return;

    // Continue until a bracket or identifier is found
    token_end = FindToken(token_begin, tokens_.end(),
                          kFlagBracket | kFlagIdentifier);

    // Ignore if it's only a dash
    if (std::distance(token_begin, token_end) <= 2 &&
        IsDashCharacter(token_begin->content)) {
      continue;
    }

    // Build episode title
    BuildElement(kElementEpisodeTitle, false, token_begin, token_end);
    return;
  } while (token_begin != tokens_.end());
}

void Parser::SearchForReleaseGroup() {
  token_container_t::iterator token_begin = tokens_.begin();
  token_container_t::iterator token_end = tokens_.begin();

  do {
    // Find the first enclosed unknown token
    token_begin = FindToken(token_end, tokens_.end(),
                            kFlagEnclosed | kFlagUnknown);
    if (token_begin == tokens_.end())
      return;

    // Continue until a bracket or identifier is found
    token_end = FindToken(token_begin, tokens_.end(),
                          kFlagBracket | kFlagIdentifier);
    if (token_end->category != kBracket)
      continue;

    // Ignore if it's not the first non-delimiter token in group
    token_container_t::iterator previous_token = FindPreviousToken(tokens_, token_begin,
                                            kFlagNotDelimiter);
    if (previous_token != tokens_.end() &&
        previous_token->category != kBracket) {
      continue;
    }

    // Build release group
    BuildElement(kElementReleaseGroup, true, token_begin, token_end);
    return;
  } while (token_begin != tokens_.end());
}

void TLexer::Read() {
    char c;
    std::string value = "";
    while (true) {
        c = GetChar();
        if (FileEOF()) {
            break;
        }
        value += c;
        switch (c) {
            case 34: {
                ReadString(value);
                break;
            }

            case 39: {
                ReadSymbol(value);
                break;
            }

            case '.' : {
                FindToken(State_Point, value);
                break;
            }

            case '(': {
                FindToken(State_Lbkt,value);
                break;
            }

            case ')': {
                FindToken(State_Rbkt,value);
                break;
            }
            case '#': {
                ReadSharp(value);
                break;
            }
            default: {
                if (isspecial(c) || isalpha(c)) {
                    ReadIdent(value);
                } else if (isdigit(c)) {
                    ReadNumber(value);
                } else if (IsEndToken(c)) {
                    value.erase(value.size() - 1, value.size());
                }
                break;
            }
        }
    }
    value = "EOF";
    FindToken(State_EOF, value);
}

void MenuManager::ParsePageInfo(char* plinePtr, int* menuNumber, int* pageNumber, int* itemNumber)
{	
	(*pageNumber)++;
	*itemNumber = -1;

	char	pdrawColor[20];
	ULONG color;
	char	pseparators[] = {0x20, 0x2c, 0x3d, 0x3b, 0x0d, 0x0a, 0x09, 0x23, 0x00};
	char	*ptoken;

	PageStruct* pPage = &(mpMenus[*menuNumber].mpPages[*pageNumber]);

	#ifdef USE_SH_POOLS
	pPage->mpTitle = (char *)MemAllocPtr(gCockMemPool,sizeof(char)*mMaxTextLen,FALSE);
	#else
	pPage->mpTitle		= new char[mMaxTextLen];
	#endif

	*(pPage->mpTitle)	= '\0';

	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%d",  &(pPage->mNumItems));
	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%d",  &(pPage->mCondition));
	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%s",  pdrawColor);

	*(pPage->mpTitle) = NULL;
	ptoken = FindToken(&plinePtr, pseparators);	
	while(ptoken) {
		strcat(pPage->mpTitle, ptoken);
		ptoken = FindToken(&plinePtr, pseparators);
		if(ptoken) {
			strcat(pPage->mpTitle, " ");
		}
	}

	if(FindMenuColorValue(pdrawColor, &color)) {
		pPage->mDrawColor = color;
	}
	else {
		pPage->mDrawColor = gMenuColorTable[MENU_WHITE].mValue;
	}


	#ifdef USE_SH_POOLS
	pPage->mpItems = (ItemStruct *)MemAllocPtr(gCockMemPool,sizeof(ItemStruct)*pPage->mNumItems,FALSE);
	#else
	pPage->mpItems = new ItemStruct[pPage->mNumItems];
	#endif
}

L_TOKEN* IDENTIFIER(char*          source,
                    int            start,
                    int            end,
                    int            line,
                    GRAMMAR_TABLE  grammar)
{
    int token;
    L_TOKEN* block = (L_TOKEN*)malloc(sizeof(L_TOKEN));

    token = FindToken(&source[start], end - start, grammar);
    if (token) {
        // token
        block->token = token;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    } else {
        // identifier
        block->token = gSymbolIdentifier;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    }

    return block;
}

L_TOKEN* TOKEN(char*          source,
               int            start,
               int            end,
               int            line,
               GRAMMAR_TABLE  grammar)
{
    int token;
    L_TOKEN* block = (L_TOKEN*)malloc(sizeof(L_TOKEN));

    token = FindToken(&source[start], end - start, grammar);
    if (token) {
        // token
        block->token = token;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    } else {
        // identifier
        free(block);
        block = NULL;
        printf("Syntax error on line %i, ", line+1);
        printf("Error, illegal identifier: ");
        while (start < end)
            printf("0x%X", source[start++]);
        printf("\n");
    }

    return block;
}

token_iterator_t FindPreviousToken(token_container_t& tokens,
                                   token_iterator_t first,
                                   unsigned int flags) {
  auto it = FindToken(std::reverse_iterator<token_iterator_t>(first),
                       tokens.rend(), flags);
  return it == tokens.rend() ? tokens.end() : (++it).base();
}

int StyleSheetParser::Parse(StyleSheetNode* node, Stream* _stream)
{
	int rule_count = 0;
	line_number = 0;
	stream = _stream;
	stream_file_name = stream->GetSourceURL().GetURL().Replace("|", ":");

	// Look for more styles while data is available
	while (FillBuffer())
	{
		String style_names;
		
		while (FindToken(style_names, "{", true))
		{
			// Read the attributes
			PropertyDictionary properties;
			if (!ReadProperties(properties))
			{
				continue;
			}

			StringList style_name_list;
			StringUtilities::ExpandString(style_name_list, style_names);

			// Add style nodes to the root of the tree
			for (size_t i = 0; i < style_name_list.size(); i++)
				ImportProperties(node, style_name_list[i], properties, rule_count);

			rule_count++;
		}
	}	

	return rule_count;
}

void pgSettingItem::SetType(const wxString &str)
{
	int index = FindToken(str, pgConfigTypeStrings);
	if (index < 0)
		type = PGC_STRING;
	else
		type = (pgConfigType)index;
}

void TLexer::IsCorrectToken(char c, std::string& value, States state) {
    if (IsEndToken(c)) {
        FindToken(state, value);
    } else {
        value += c;
        throw new ExceptionLexerIncorrectChar(value, PosLine, PosColumn);
    }
}

struct TokenStruct NextToken (char *c) {
	*c = FindToken (*c);
	struct TokenStruct ret;// = malloc (sizeof (struct TokenStruct));
	ret.Token = malloc (sizeof (char) * 2);
	ret.Token[0] = *c;
	ret.type = TokenType (*c);
	ret.priority = OpPriority (*c);
	ret.index = line_index;
	
	int length = 1;
	if (!ret.type) {
		if (*c == '+' || *c == '-' || *c == '*' || *c == '/' || *c == '%' || *c == '^' || *c == '=' || *c == ')' || *c == '(' || *c == ',') {	// Grab next character if we have a valid token
			*c = FindToken (0);
		}
		ret.Token[1] = '\0';
#ifdef DEBUG
		if (ret.Token[0] != '\n') { printf ("	Op/Invalid Token %i: '%s';\n", ret.index, ret.Token); }
		else { printf ("	Op/Invalid Token %i: '\\n';\n", ret.index); }
		sleep (1);
#endif
		return ret;
	}
	
	*c = getchar ();
	AddSpace (*c);
	while (TokenType (*c) == ret.type) {
		ret.Token[length++] = *c;
		if (ECHO) { putchar (*c); }
		ret.Token = realloc (ret.Token, sizeof (char) * (length+1));
		*c = getchar ();
		AddSpace (*c);
	}
	if (ECHO) { putchar (*c); }
	//line_index += length;
	ret.Token[length] = '\0';
	*c = FindToken (*c);
#ifdef DEBUG
	printf ("	Var/Num Token %i: %s;\n", ret.index, ret.Token);
	sleep (1);
#endif
	return ret;
}

void MenuManager::ReadDataFile(char* pfileName)
{
	FILE*				pFile;
	BOOL				quitFlag  = FALSE;
	char*				presult = "";
	char*				plinePtr;
	const int		lineLen = MAX_LINE_BUFFER - 1;
	char				plineBuffer[MAX_LINE_BUFFER] = "";
	char				*ptoken;
	char				pseparators[] = {0x20, 0x2c, 0x3d, 0x3b, 0x0d, 0x0a, 0x09, 0x23, 0x00};
	int				menuNumber = -1;
	int				pageNumber = -1;
	int				itemNumber = -1;

	pFile = CP_OPEN(pfileName, "r");
	F4Assert(pFile);

	presult	= fgets(plineBuffer, lineLen, pFile);
	quitFlag	= (presult == NULL);
	plinePtr = plineBuffer;

	while(!quitFlag) {

		if(*plineBuffer == '#') {

			ptoken = FindToken(&plinePtr, pseparators);	

			if(!strcmpi(ptoken, TYPE_MENUMANGER_STR)) {
				ParseManagerInfo(plinePtr);
			}
			else if(!strcmpi(ptoken, TYPE_MENU_STR)) {
				ParseMenuInfo(plinePtr, &menuNumber, &pageNumber);
			}
			else if(!strcmpi(ptoken, TYPE_PAGE_STR)) {
				ParsePageInfo(plinePtr, &menuNumber, &pageNumber, &itemNumber);
			}
			else if(!strcmpi(ptoken, TYPE_ITEM_STR)) {
				ParseItemInfo(plinePtr, &menuNumber, &pageNumber, &itemNumber);
			}
			else if(!strcmpi(ptoken, TYPE_RES_STR)) {
				ParseResInfo(plinePtr, --mResCount);
			}
			else {
				ShiWarning ("Bad String in menu file");
			}
		}
		presult	= fgets(plineBuffer, lineLen, pFile);
		plinePtr = plineBuffer;
		quitFlag	= (presult == NULL);
	}

	CP_CLOSE(pFile);
	ShiAssert(mResCount == 0);	// Bad data in the manager information line!
}

void PdfParser::ReadXRef( pdf_long* pXRefOffset )
{
    FindToken( "startxref", PDF_XREF_BUF );

    if( !this->IsNextToken( "startxref" ) )
    {
        PODOFO_RAISE_ERROR( ePdfError_NoXRef );
    }

    *pXRefOffset = this->GetNextNumber();
}

void ExplicitConstructorCheck::check(const MatchFinder::MatchResult &Result) {
  const CXXConstructorDecl *Ctor =
      Result.Nodes.getNodeAs<CXXConstructorDecl>("ctor");
  // Do not be confused: isExplicit means 'explicit' keyword is present,
  // isImplicit means that it's a compiler-generated constructor.
  if (Ctor->isOutOfLine() || Ctor->isImplicit() || Ctor->isDeleted() ||
      Ctor->getNumParams() == 0 || Ctor->getMinRequiredArguments() > 1)
    return;

  bool takesInitializerList = isStdInitializerList(
      Ctor->getParamDecl(0)->getType().getNonReferenceType());
  if (Ctor->isExplicit() &&
      (Ctor->isCopyOrMoveConstructor() || takesInitializerList)) {
    auto isKWExplicit = [](const Token &Tok) {
      return Tok.is(tok::raw_identifier) &&
             Tok.getRawIdentifier() == "explicit";
    };
    SourceRange ExplicitTokenRange =
        FindToken(*Result.SourceManager, Result.Context->getLangOpts(),
                  Ctor->getOuterLocStart(), Ctor->getLocEnd(), isKWExplicit);
    StringRef ConstructorDescription;
    if (Ctor->isMoveConstructor())
      ConstructorDescription = "move";
    else if (Ctor->isCopyConstructor())
      ConstructorDescription = "copy";
    else
      ConstructorDescription = "initializer-list";

    DiagnosticBuilder Diag =
        diag(Ctor->getLocation(),
             "%0 constructor should not be declared explicit")
        << ConstructorDescription;
    if (ExplicitTokenRange.isValid()) {
      Diag << FixItHint::CreateRemoval(
          CharSourceRange::getCharRange(ExplicitTokenRange));
    }
    return;
  }

  if (Ctor->isExplicit() || Ctor->isCopyOrMoveConstructor() ||
      takesInitializerList)
    return;

  bool SingleArgument =
      Ctor->getNumParams() == 1 && !Ctor->getParamDecl(0)->isParameterPack();
  SourceLocation Loc = Ctor->getLocation();
  diag(Loc,
       "%0 must be marked explicit to avoid unintentional implicit conversions")
      << (SingleArgument
              ? "single-argument constructors"
              : "constructors that are callable with a single argument")
      << FixItHint::CreateInsertion(Loc, "explicit ");
}

char const *FindInitialToken(Token *tok, char const *s)
{
    DynamicBuffer buf;
    DBufInit(&buf);

    tok->type = T_Illegal;

    while (isempty(*s)) s++;

    while (*s && !isempty(*s)) {
	if (DBufPutc(&buf, *s++) != OK) return s;
    }

    FindToken(DBufValue(&buf), tok);
    DBufFree(&buf);

    return s;
}

ULONG tScriptState::ReadCommand(CString& s)
{
    char buf[256];
    if (!fgets(buf, sizeof(buf) - 1, f))
    {
        if (feof(f)) s = FindToken(cmdEnd);
        else s = "<Error reading from file>";
    }
    else
    {
        s = buf;
    }
    while (isspace(s[0])) s.Delete(0, 1);
    s.Remove('\n');
    s.Remove('\r');
    currentLine++;
    currentDot = 0;
    strcpy(buf, s);
    return currentCommandIndex++;
}

/*
 * Routine: ProcessTypes
 * Purpose: Parse the type vector
 * Algorithm:
 * Data Structures:
 *
 * Params:
 * Returns:
 * Called By:
 * Calls:
 * Assumptions:
 * Side Effects:
 * TODO: None
 */
int
ProcessTypes (char *stmt, token_t * tokens)
{
    char *cp,
         *cp1;
    int nTypeCount = 1,
        nToken,
        i;

    /* get a type count */
    for (cp1 = stmt; (cp1 = strchr (cp1, ',')) != NULL; cp1++)
        nTypeCount += 1;
    pCurrentIndexEntry->dist->type_vector =
        (int *) MALLOC (sizeof (int) * nTypeCount);
    if (pCurrentIndexEntry->dist->type_vector == NULL)
        return (QERR_NO_MEMORY);
    memset(pCurrentIndexEntry->dist->type_vector, 0, sizeof(int) * nTypeCount);

    /* get the type names */
    i = 0;
    while ((cp = strtok (NULL, "=( ,);")) != NULL)
    {
        switch (nToken = FindToken (cp))
        {
        /*
         * NOTE NOTE NOTE NOTE NOTE
         * this is manually sync'd with expr.h values
         * NOTE NOTE NOTE NOTE NOTE
         */
        case TKN_INT:
        case TKN_VARCHAR:
            pCurrentIndexEntry->dist->type_vector[i++] = nToken;
            break;
        default:
            return (QERR_SYNTAX);
        }
    }

    return (nTypeCount);
}

bool cStrings::UnderstandToken(int token, char *data)
{
	sFlagPair		*FlagPair;

	switch(token)
	{
		case TK_FLAGS:
			while(token != TK_INVALID)
			{
				token = FindToken(data, false);
				for(FlagPair = FlagPairs; FlagPair->Name != TK_INVALID; FlagPair++)
				{
					if (FlagPair->Name == token)
					{
						Flags |= FlagPair->Value;
						break;
					}
				}
			}
			return true;

		case TK_REFERENCE:
			SetReference(data);
			return true;

		case TK_RIGHT_BRACE:
			return false;
	}

	if (token == TK_INVALID)
	{
		return false;
	}

	return true;
}

/* handle .model directive
 * syntax: .MODEL <FLAT|TINY|SMALL...> [,<C|PASCAL|STDCALL...>][,<NEARSTACK|FARSTACK>][,<OS_DOS|OS_OS2>]
 * sets fields
 * - ModuleInfo.model
 * - ModuleInfo.language
 * - ModuleInfo.distance
 * - ModuleInfo.ostype
 * if model is FLAT, defines FLAT pseudo-group
 * set default segment names for code and data
 */
ret_code ModelDirective( int i, struct asm_tok tokenarray[] )
/***********************************************************/
{
    enum model_type model;
    enum lang_type language;
    enum dist_type distance;
    enum os_type ostype;
    int index;
    uint_8 init;
    uint_8 initv;

    DebugMsg1(("ModelDirective enter\n"));
    /* v2.03: it may occur that "code" is defined BEFORE the MODEL
     * directive (i.e. DB directives in AT-segments). For FASTPASS,
     * this may have caused errors because contents of the ModuleInfo
     * structure was saved before the .MODEL directive.
     */
    //if( Parse_Pass != PASS_1 ) {
    if( Parse_Pass != PASS_1 && ModuleInfo.model != MODEL_NONE ) {
        /* just set the model with SetModel() if pass is != 1.
         * This won't set the language ( which can be modified by
         * OPTION LANGUAGE directive ), but the language in ModuleInfo
         * isn't needed anymore once pass one is done.
         */
        SetModel();
        return( NOT_ERROR );
    }

    i++;
    if ( tokenarray[i].token == T_FINAL ) {
        EmitError( EXPECTED_MEMORY_MODEL );
        return( ERROR );
    }
    /* get the model argument */
    index = FindToken( tokenarray[i].string_ptr, ModelToken, sizeof( ModelToken )/sizeof( ModelToken[0] ) );
    if( index >= 0 ) {
        if( ModuleInfo.model != MODEL_NONE ) {
            EmitWarn( 2, MODEL_DECLARED_ALREADY );
            return( NOT_ERROR );
        }
        model = index + 1;
        i++;
    } else {
        EmitErr( SYNTAX_ERROR_EX, tokenarray[i].string_ptr );
        return( ERROR );
    }

    /* get the optional arguments: language, stack distance, os */
    init = 0;
    while ( i < ( Token_Count - 1 ) && tokenarray[i].token == T_COMMA ) {
        i++;
        if ( tokenarray[i].token != T_COMMA ) {
            if ( GetLangType( &i, tokenarray, &language ) == NOT_ERROR ) {
                initv = INIT_LANG;
            } else {
                index = FindToken( tokenarray[i].string_ptr, ModelAttr, sizeof( ModelAttr )/sizeof( ModelAttr[0] ) );
                if ( index < 0 )
                    break;
                initv = ModelAttrValue[index].init;
                switch ( initv ) {
                case INIT_STACK:
                    if ( model == MODEL_FLAT ) {
                        EmitError( INVALID_MODEL_PARAM_FOR_FLAT );
                        return( ERROR );
                    }
                    distance = ModelAttrValue[index].value;
                    break;
                case INIT_OS:
                    ostype = ModelAttrValue[index].value;
                    break;
                }
                i++;
            }
            /* attribute set already? */
            if ( initv & init ) {
                i--;
                break;
            }
            init |= initv;
        }
    }
    /* everything parsed successfully? */
    if ( tokenarray[i].token != T_FINAL ) {
        EmitErr( SYNTAX_ERROR_EX, tokenarray[i].tokpos );
        return( ERROR );
    }

    if ( model == MODEL_FLAT ) {
        if ( ( ModuleInfo.curr_cpu & P_CPU_MASK) < P_386 ) {
            EmitError( INSTRUCTION_OR_REGISTER_NOT_ACCEPTED_IN_CURRENT_CPU_MODE );
//.........这里部分代码省略.........