main()
{
    symtabADT table;
    scannerADT scanner;
    string line, name, value;

    table = NewSymbolTable();
    scanner = NewScanner();
    SetScannerSpaceOption(scanner, IgnoreSpaces);
    while (TRUE) {
        printf("-> ");
        line = GetLine();
        if (StringEqual(line, "quit")) break;
        SetScannerString(scanner, line);
        name = ReadToken(scanner);
        if (MoreTokensExist(scanner)) {
            if (!StringEqual(ReadToken(scanner), "=")) {
                Error("Missing equal sign in assignment");
            }
            value = ReadToken(scanner);
            if (MoreTokensExist(scanner)) {
                Error("Statement contains additional data");
            }
            Enter(table, name, value);
        } else {
            value = Lookup(table, name);
            if (value == UNDEFINED) {
                printf("'%s' is undefined.\n", name);
            } else {
                printf("%s\n", value);
            }
        }
    }
    FreeSymbolTable(table);
}

main()
{
    dataBlockT db;
    string line, cmd, token, var;
    int pos;
    printf("List structure test program\n");
    printf("Type \"help\" for help\n");
    db = NewDataBlock();
    while (TRUE) {
        printf(">");
        line = GetLine();
        SetScannerString(db->scanner, line);
        cmd = ConvertToLowerCase(ReadToken(db->scanner));
        if (IsVariableName(cmd)) {
            var = cmd;
            token = ReadToken(db->scanner);
            if (StringEqual(token, "")) {
                DisplayStringList(GetValue(db, var));
            } else if (StringEqual(token, "=")) {
                ParseOperation(db, var);
            } else {
                Error("Unexpected token %s", token);
            }
        } else if (StringEqual(cmd, "help")) {
            HelpCmd();
        } else if (StringEqual(cmd, "quit")) {
            break;
        } else {
            SaveToken(db->scanner, cmd);
            ParseOperation(db, NULL);
        }
    }
}

//===========================================================================
// DoKeyword
//	The next token is expected to be a keyword.
//===========================================================================
int DoKeyword(void)
{
	ReadToken();
	if(ISTOKEN("syntax"))
	{
		ReadToken();
		if(ISTOKEN("simple"))
		{
			syntax = STX_SIMPLE;
			Message("Using simple syntax.");
		}
		else
		{
			Message("Unknown syntax '%s'.", token);
			return false;
		}
	}
	else if(ISTOKEN("group"))
	{
		ReadToken();
		group = strtol(token, 0, 0);
		if(group < 1 || group > NUM_GROUPS)
		{
			Message("Illegal group number %i (1..%i allowed).", group,
				NUM_GROUPS);
			return false;
		}
		Message("Switching to group %i.", group);
		group--;	// Make it an index number, though.
	}
	return true;
}

int HandleCRAMMD5( int inSocket )
{
	unsigned char	pChallenge[255]	= { 0, };
	char			pHostname[128]	= { 0, };
	char			*pResponse		= NULL;
	char			*pUsername		= NULL;
	struct timeval	stCurrentTime;
	int				iResult			= -1;

	// we will be using ReadToken and WriteToken to send data back and forth for this
	// example, though not necessarily the most efficient, it is simplistic for this 
	// demonstration.
	
	// Since CRAM-MD5 was requested, let's generate a challenge and send it to the client
	// using example method in RFC 1460, page 12.
	gethostname( pHostname, 127 );
	gettimeofday( &stCurrentTime, NULL ); // assume no error occurred
	snprintf( pChallenge, 255, "<%ld.%[email protected]%s>", (long) getpid(), stCurrentTime.tv_sec, pHostname );
	
	printf( "Sending challenge %s\n", pChallenge );
	
	// send our challenge to the client
	if( SendToken(inSocket, pChallenge, strlen(pChallenge)) > 0 ) {
		
		// now wait for username and response to return
		if( ReadToken(inSocket, &pUsername) > 0 ) {
			
			if( ReadToken(inSocket, &pResponse) > 0 ) {

				// here is where we authenticate the user using Open Directory
				iResult = AuthCRAMMD5( pUsername, pChallenge, pResponse );
				
				// send a response
				if( iResult == eDSNoErr ) {
					SendToken( inSocket, "Accepted", sizeof("Accepted") );
				} else {
					SendToken( inSocket, "Rejected", sizeof("Rejected") );
				}
			}
		}
	}
	
	// free up any strings we allocated
	if( pUsername != NULL ) {
		free( pUsername );
		pUsername = NULL;
	}

	if( pResponse != NULL ) {
		free( pResponse );
		pResponse = NULL;
	}

	return iResult;
}

static void SelectionOperation(dataBlockT db, string lhs)
{
    string var;
    int pos;

    if (lhs != NULL) Error("NthElement result cannot be assigned");
    CheckForToken(db->scanner, "(");
    var = ConvertToLowerCase(ReadToken(db->scanner));
    CheckForToken(db->scanner, ",");
    pos = StringToInteger(ReadToken(db->scanner));
    CheckForToken(db->scanner, ")");
    printf("%s\n", NthElement(GetValue(db, var), pos));
}

int HandleCleartext( int inSocket )
{
	char	*pUsername		= NULL;
	char	*pPassword		= NULL;
	char	*pRecordname	= NULL;
	int		iResult			= -1;
	
	// we will be using ReadToken and WriteToken to send data back and forth for this
	// example, though not necessarily the most efficient, it is simplistic for this 
	// demonstration.
	
	// we expect 2 tokens, username then password
	
	// read the username from the network stream
	if( ReadToken(inSocket, &pUsername) > 0 ) {
		
		// read the password from the network stream
		if( ReadToken(inSocket, &pPassword) > 0 ) {
			
			iResult = AuthCleartext( pUsername, pPassword );
			
			// send a response
			if( iResult == eDSNoErr ) {
				SendToken( inSocket, "Accepted", sizeof("Accepted") );
			} else {
				SendToken( inSocket, "Rejected", sizeof("Rejected") );
			}
		}
	}
	
	// free any allocated strings
	if( pUsername != NULL ) {
		free( pUsername );
		pUsername = NULL;
	}
	
	if( pPassword != NULL ) {
		free( pPassword );
		pPassword = NULL;
	}

	if( pRecordname != NULL ) {
		free( pRecordname );
		pRecordname = NULL;
	}
	
	return iResult;
}

void test(void)
{
	char buf[128];
	xprintf(0x21, "Hello World!\r\n");
	ReadToken(buf, TOKEN_PASSWORD, 8);
	xprintf(0x21, "password = '%s'\r\n", buf);
}

/*
=================
idLexer::SkipBracedSection

Skips until a matching close brace is found.
Internal brace depths are properly skipped.
=================
*/
int idLexer::SkipBracedSection( bool parseFirstBrace )
{
	idToken token;
	int depth;
	
	depth = parseFirstBrace ? 0 : 1;
	do
	{
		if( !ReadToken( &token ) )
		{
			return false;
		}
		if( token.type == TT_PUNCTUATION )
		{
			if( token == "{" )
			{
				depth++;
			}
			else if( token == "}" )
			{
				depth--;
			}
		}
	}
	while( depth );
	return true;
}

void ParsedObject::MatchToken(const LispString* aToken)
{
    if (aToken != iLookAhead)
        Fail();

    ReadToken();
}

/*
================
Lexer::ReadFloat
================
*/
float Lexer::ReadFloat( void ) {
	if ( !ReadToken() )
		throw LexerError( LexerError::END_OF_FILE );
	if ( !String::IsNumeric( token.GetString() ) )
		throw LexerError( LexerError::BAD_TOKEN, token.line, "float", token.GetString() );
	return String::ToFloat( token.GetString() );
}

static TokenStream *PrescanMacroArg(TokenStream *a, yystypepp * yylvalpp)
{
	int token;
	TokenStream *n;
	RewindTokenStream(a);
	do {
		token = ReadToken(a, yylvalpp);
		if (token == CPP_IDENTIFIER && LookUpSymbol(macros, yylvalpp->sc_ident))
			break;
	} while (token > 0);
	if (token <= 0)
		return a;
	n = NewTokenStream("macro arg", 0);
	PushEofSrc();
	ReadFromTokenStream(a, 0, 0);
	while ((token = cpp->currentInput->scan(cpp->currentInput, yylvalpp)) > 0) {
		if (token == CPP_IDENTIFIER
				&& MacroExpand(yylvalpp->sc_ident, yylvalpp))
			continue;
		RecordToken(n, token, yylvalpp);
	}
	PopEofSrc();
	DeleteTokenStream(a);
	return n;
} /* PrescanMacroArg */

TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken* ppToken)
{
    int token;
    TokenStream *n;
    RewindTokenStream(a);
    do {
        token = ReadToken(a, ppToken);
        if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom))
            break;
    } while (token != EOF);
    if (token == EOF)
        return a;
    n = new TokenStream;
    PushEofSrc();
    ReadFromTokenStream(a, 0, 0);
    while ((token = currentInput->scan(this, currentInput, ppToken)) > 0) {
        if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, 0) == 1)
            continue;
        RecordToken(n, token, ppToken);
    }
    PopEofSrc();
    delete a;

    return n;
}

bool FFeedbackContextMarkup::ParseCommand(const FString& Line, FFeedbackContext* Warn)
{
	const TCHAR *Text = *Line;
	if(ReadToken(Text, TEXT("@progress")))
	{
		FString Status;
		bool bHaveStatus = ReadString(Text, Status);

		int32 Numerator, Denominator;
		bool bHaveProgress = ReadProgress(Text, Numerator, Denominator);

		if(*Text == 0)
		{
			if(bHaveProgress && bHaveStatus)
			{
				Warn->StatusUpdate(Numerator, Denominator, FText::FromString(Status));
				return true;
			}
			if(bHaveProgress)
			{
				Warn->UpdateProgress(Numerator, Denominator);
				return true;
			}
		}
	}
	return false;
}

bool IACFile::Read(wxInputStream &stream) {
    bool isok = false;  // true if minimum one token was read from file
    Invalidate();
    wxString token;
    m_tokensI = 0;
    if (stream.IsOk()) {
        for (;;) {
            token = ReadToken(stream);
            if (!token.IsEmpty()) {
                m_tokens.Add(token);
                m_tokensI++;
                isok = true;
            } else {
                break;
            }
        }
    }
    m_tokensI = 0;

    //    for (std::vector<size_t>::iterator it = m_newlineTokens.begin(); it != m_newlineTokens.end(); ++it)
    //    {
    //        wxMessageBox( wxString::Format( _T("ID: %i :"), *it ) + m_tokens[*it] );
    //    }

    if (isok) {
        // decode tokens if some were found
        isok = Decode();
    }
    m_isok = isok;
    return isok;
}

/*************************************************************
 * Que 3. Post fix expression evaluation
 * 
 * Comment    : 1) prepare stack for holding operands only.
 *              2) Push when numbers are read.
 *              3) Pop two numbers when ever operand is encountered.
 * 
 * Parameters : expression array
 * Returns    : int (eval result)
 *************************************************************/
int  PostFixEval(char A[])
{
   LinkedListStack stack;
   int tokenType = EMPTY,num,i=0;;
   char token;

   while((tokenType = ReadToken(A,i,token,num)) != EMPTY)
   {
      if(tokenType == OPERAND) //Numbers
      {
	 stack.Push(token - '0');
      }
      else if(tokenType == OPERATOR) //Mathematical operator
      {
	int A = stack.Pop();
	int B = stack.Pop();
	stack.Push(Operate(A,B,token));
      }
      else 
      {
	//Should not reach here
	assert(1 == 0);
      }
  }
  if(stack.IsEmpty())
    return -1;
  return stack.Pop();   
}

/* macro_scan ---
 ** return the next token for a macro expanion, handling macro args
 */
static int macro_scan(MacroInputSrc *in, yystypepp * yylvalpp)
{
	int i;
	int token = ReadToken(in->mac->body, yylvalpp);
	if (token == CPP_IDENTIFIER) {
		for (i = in->mac->argc - 1; i >= 0; i--)
			if (in->mac->args[i] == yylvalpp->sc_ident)
				break;
		if (i >= 0) {
			ReadFromTokenStream(in->args[i], yylvalpp->sc_ident, 0);
			return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
		}
	}
	if (token > 0)
		return token;
	in->mac->busy = 0;
	cpp->currentInput = in->base.prev;
	if (in->args) {
		for (i = in->mac->argc - 1; i >= 0; i--)
			DeleteTokenStream(in->args[i]);
		free(in->args);
	}
	free(in);
	return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
} /* macro_scan */

TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
{
    int token;
    TokenStream *n;
    RewindTokenStream(a);
    do {
        token = ReadToken(a, ppToken);
        if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
            break;
    } while (token != EndOfInput);

    if (token == EndOfInput)
        return a;

    n = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(a);
    while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
        if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
            continue;
        RecordToken(n, token, ppToken);
    }
    popInput();
    delete a;

    return n;
}

	bool ParseTag(FColor& OutColor)
	{
		FString TagString;

		EStringParserToken Token = ReadToken();
		for (; Token != EStringParserToken::EndOfString && Token != EStringParserToken::CloseTag; Token = ReadToken())
		{
			if (Token == EStringParserToken::RegularChar)
			{
				TagString.AppendChar(DataString[DataIndex]);
			}
			
			DataIndex++;
		}

		bool bResult = false;
		if (Token == EStringParserToken::CloseTag)
		{
			const FString TagColorLower = TagString.ToLower();
			const bool bIsColorName = GColorList.IsValidColorName(*TagColorLower);
			
			if (bIsColorName)
			{
				OutColor = GColorList.GetFColorByName(*TagColorLower);
				bResult = true;
			}
			else
			{
				bResult = OutColor.InitFromString(TagString);
			}
		}

		return bResult;
	}

char *ReadToken (server_request *ServerRequest)
{
	char *c = &ServerRequest->Request[ServerRequest->TokenizerIndex];
	if (!*c)
	{
		return NULL;
	}
	char *Start = c;
	s32 Length = 0;
	while (*c != 0 &&
		   *c != ' ' &&
		   *c != '\n' &&
		   *c != '\r')
	{
		++Length;
		++c;
	}

	if (!Length)
	{
		++ServerRequest->TokenizerIndex;
		return ReadToken(ServerRequest);
	}
	
	ServerRequest->TokenizerIndex += Length;

	char *Token = PushMemory(Length + 1);
	strncpy(Token, Start, Length);
	Token[Length] = 0;
	return Token;
}

bool JsonString::ReadValue()  
{  
	Token token;  
	ReadToken(token);  
	bool successful = true;  

	switch(token.type_)  
	{  
	case tokenObjectBegin:  
		objnum++;  
		successful = ReadObject(token);  
		break;  
	case tokenArrayBegin:  
		successful = ReadArray(token);  
		break;  
	case tokenNumber:  
	case tokenString:  
	case tokenTrue:  
	case tokenFalse:  
	case tokenNull:  
		break;  
	default:  
		return false;  
	}  
	return successful;  
}