static sl_node_base_t*
statement(sl_parse_state_t* ps)
{
    sl_node_base_t* node;
    switch(peek_token(ps)->type) {
        case SL_TOK_CLOSE_TAG:
            next_token(ps);
            node = inline_raw(ps);
            if(peek_token(ps)->type != SL_TOK_END) {
                expect_token(ps, SL_TOK_OPEN_TAG);
            }
            return node;
        case SL_TOK_SEMICOLON:
            next_token(ps);
            return NULL;
        case SL_TOK_IF:
        case SL_TOK_UNLESS:
            return if_expression(ps);
        case SL_TOK_FOR:
            return for_expression(ps);
        case SL_TOK_WHILE:
        case SL_TOK_UNTIL:
            return while_expression(ps);
        default:
            node = expression(ps);
            if(peek_token(ps)->type != SL_TOK_CLOSE_TAG
                && peek_token(ps)->type != SL_TOK_CLOSE_BRACE
                && token(ps)->type != SL_TOK_CLOSE_BRACE
                && peek_token(ps)->type != SL_TOK_END) {
                expect_token(ps, SL_TOK_SEMICOLON);
            }
            return node;
    }
}

static sl_node_base_t*
send_with_args_expression(sl_parse_state_t* ps, sl_node_base_t* recv, SLID id)
{
    size_t argc = 0, cap = 2;
    sl_node_base_t** argv = sl_alloc(ps->vm->arena, sizeof(sl_node_base_t*) * cap);
    bool splat_last = false;
    expect_token(ps, SL_TOK_OPEN_PAREN);
    while(peek_token(ps)->type != SL_TOK_CLOSE_PAREN) {
        if(argc >= cap) {
            cap *= 2;
            argv = sl_realloc(ps->vm->arena, argv, sizeof(sl_node_base_t*) * cap);
        }
        if(peek_token(ps)->type == SL_TOK_TIMES) {
            next_token(ps);
            splat_last = true;
            argv[argc++] = expression(ps);
            break;
        }
        argv[argc++] = expression(ps);
        if(peek_token(ps)->type != SL_TOK_CLOSE_PAREN) {
            expect_token(ps, SL_TOK_COMMA);
        }
    }
    expect_token(ps, SL_TOK_CLOSE_PAREN);
    return sl_make_send_node(ps, recv, id, argc, argv, splat_last);
}

static sl_node_base_t*
if_expression(sl_parse_state_t* ps)
{
    sl_node_base_t *condition, *if_true, *if_false = NULL;
    int negate_condition = 0;
    if(peek_token(ps)->type == SL_TOK_ELSIF) {
        expect_token(ps, SL_TOK_ELSIF);
    } else if(peek_token(ps)->type == SL_TOK_UNLESS) {
        expect_token(ps, SL_TOK_UNLESS);
        negate_condition = 1;
    } else {
        expect_token(ps, SL_TOK_IF);
    }
    condition = expression(ps);
    if(negate_condition) {
        condition = sl_make_unary_node(ps, condition, SL_NODE_NOT);
    }
    if_true = body_expression(ps);
    if(peek_token(ps)->type == SL_TOK_ELSIF) {
        if_false = if_expression(ps);
    }
    if(peek_token(ps)->type == SL_TOK_ELSE) {
        next_token(ps);
        if_false = body_expression(ps);
    }
    return sl_make_if_node(ps, condition, if_true, if_false);
}

static sl_node_base_t*
switch_expression(sl_parse_state_t* ps)
{
    expect_token(ps, SL_TOK_SWITCH);
    sl_node_base_t* value = expression(ps);
    expect_token(ps, SL_TOK_OPEN_BRACE);
    size_t case_count = 0, case_cap = 2;
    sl_node_switch_case_t* cases = sl_alloc(ps->vm->arena, sizeof(sl_node_switch_case_t) * case_cap);
    sl_node_base_t* else_body = NULL;
    while(peek_token(ps)->type != SL_TOK_CLOSE_BRACE) {
        if(peek_token(ps)->type == SL_TOK_ELSE) {
            next_token(ps);
            else_body = body_expression(ps);
            break;
        }
        if(case_count + 1 >= case_cap) {
            case_cap *= 2;
            cases = sl_realloc(ps->vm->arena, cases, sizeof(sl_node_switch_case_t) * case_cap);
        }
        cases[case_count].value = expression(ps);
        cases[case_count].body = body_expression(ps);
        case_count++;
    }
    expect_token(ps, SL_TOK_CLOSE_BRACE);
    return sl_make_switch_node(ps, value, case_count, cases, else_body);
}

/*----------------------------------------------------------------------*/
static ast_expression_t *parse_function_call(parse_state_t *p) {
    token_t *token;
    ast_expression_t *expression;
    ast_expression_list_t *param, *new_param;

    expression = ALLOC(sizeof(ast_expression_t));
    BZERO(expression);

    expression->tag = AST_EXPRESSION_FUNCTION_CALL;
    /* function identifier */
    token = next_token(p);
    EXPECT(token, TK_IDENTIFIER, "function name expected");
    expression->u.function_call.identifier = token;

    /* opening bracket */
    token = next_token(p);
    EXPECT(token, TK_LBRACKET, "'(' expected");

    /* parameter list */
    param = NULL;
    expression->u.function_call.parameter_count = 0;
    for (;;) {
        token = peek_token(p);
        if (test_token(token, TK_RBRACKET))
            break;

        /* parameter */
        new_param = ALLOC(sizeof(ast_expression_list_t));
        new_param->next = NULL;
        new_param->expr = parse_logical_expression(p);
        if (new_param->expr == NULL)
            error(p, "expression expected");
        if (param == NULL) {
            expression->u.function_call.parameter_expr_list = new_param;
        } else {
            param->next = new_param;
        }
        param = new_param;
        expression->u.function_call.parameter_count += 1;

        /* comma? */
        token = peek_token(p);
        if (test_token(token, TK_RBRACKET))
            break;

        if (test_token(token, TK_COMMA)) {
            next_token(p);
            continue;
        }

        error(p, "',' or ')' expected");
    }

    /* closing bracket */
    token = next_token(p);
    EXPECT(token, TK_RBRACKET, "')' expected");

    return expression;
}

/* Consumes tokens from source until the next character is neither incrementer
   nor decrementer.  Returns the number of incrementer occurrences minus the
   number of decrementer occurrences. */
static int collapse(FILE* source, char decrementer, char incrementer)
{
	int net_change = 0;
	while (peek_token(source) == decrementer || peek_token(source) == incrementer)
	{
		if (get_token(source) == decrementer) net_change--;
		else net_change++;
	}
	return net_change;
}

Token Lexer::get_expected_token(TokenType tt, const string& raw)
{
    TokenType p = peek_token().type;
    string s = peek_token().as_string();
    if (p != tt && s != raw) {
        string msg = S("expected ") + tokentype2str(tt) + " or `" + raw + "'";
        throw_syntax_error(p == kTokenNop ? msg
                                          : msg + " instead of `" + s + "'");
    }
    return get_token();
}

static sl_node_base_t*
call_expression(sl_parse_state_t* ps)
{
    sl_node_base_t* left = primary_expression(ps);
    sl_node_base_t** nodes;
    size_t node_len;
    size_t node_cap;
    sl_token_t* tok;
    if(left->type == SL_NODE_VAR && peek_token(ps)->type == SL_TOK_OPEN_PAREN) {
        left = send_with_args_expression(ps, sl_make_self_node(ps),
            sl_intern2(ps->vm, sl_make_ptr((sl_object_t*)((sl_node_var_t*)left)->name)));
    }
    while(1) {
        tok = peek_token(ps);
        switch(tok->type) {
            case SL_TOK_DOT:
                next_token(ps);
                left = send_expression(ps, left);
                break;
            case SL_TOK_COLON:
                next_token(ps);
                left = sl_make_bind_method_node(ps, left, def_expression_method_name(ps));
                break;
            case SL_TOK_PAAMAYIM_NEKUDOTAYIM:
                next_token(ps);
                tok = expect_token(ps, SL_TOK_CONSTANT);
                left = sl_make_const_node(ps, left,
                    sl_intern2(ps->vm,
                        sl_make_string(ps->vm, tok->as.str.buff, tok->as.str.len)));
                break;
            case SL_TOK_OPEN_BRACKET:
                next_token(ps);
                node_cap = 1;
                node_len = 0;
                nodes = sl_alloc(ps->vm->arena, sizeof(SLVAL) * node_cap);
                while(peek_token(ps)->type != SL_TOK_CLOSE_BRACKET) {
                    if(node_len >= node_cap) {
                        node_cap *= 2;
                        nodes = sl_realloc(ps->vm->arena, nodes, sizeof(SLVAL) * node_cap);
                    }
                    nodes[node_len++] = expression(ps);
                    if(peek_token(ps)->type != SL_TOK_CLOSE_BRACKET) {
                        expect_token(ps, SL_TOK_COMMA);
                    }
                }
                expect_token(ps, SL_TOK_CLOSE_BRACKET);
                left = sl_make_send_node(ps, left, sl_intern(ps->vm, "[]"), node_len, nodes);
                break;
            default:
                return left;
        }
    }
}

static sl_node_base_t*
add_expression(sl_parse_state_t* ps)
{
    sl_node_base_t* left = mul_expression(ps);
    sl_node_base_t* right;
    sl_token_t* tok;
    while(peek_token(ps)->type == SL_TOK_PLUS || peek_token(ps)->type == SL_TOK_MINUS) {
        tok = next_token(ps);
        right = mul_expression(ps);
        left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right);
    }
    return left;
}

static sl_node_base_t*
shift_expression(sl_parse_state_t* ps)
{
    sl_node_base_t* left = add_expression(ps);
    sl_node_base_t* right;
    sl_token_t* tok;
    while(peek_token(ps)->type == SL_TOK_SHIFT_LEFT || peek_token(ps)->type == SL_TOK_SHIFT_RIGHT) {
        tok = next_token(ps);
        right = add_expression(ps);
        left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right);
    }
    return left;
}

static SLID
def_expression_method_name(sl_parse_state_t* ps)
{
    switch(peek_token(ps)->type) {
        case SL_TOK_IDENTIFIER:
            return sl_intern2(ps->vm, next_token(ps)->str);
        /* operators: */
        case SL_TOK_SHIFT_LEFT:
        case SL_TOK_SHIFT_RIGHT:
        case SL_TOK_DBL_EQUALS:
        case SL_TOK_NOT_EQUALS:
        case SL_TOK_SPACESHIP:
        case SL_TOK_LTE:
        case SL_TOK_LT:
        case SL_TOK_GTE:
        case SL_TOK_GT:
        case SL_TOK_PLUS:
        case SL_TOK_POW:
        case SL_TOK_TIMES:
        case SL_TOK_DIVIDE:
        case SL_TOK_MOD:
        case SL_TOK_CARET:
        case SL_TOK_AMP:
        case SL_TOK_PIPE:
            return sl_intern2(ps->vm, next_token(ps)->str);
        /* operators that can also be unary: */
        case SL_TOK_MINUS:
        case SL_TOK_TILDE:
        {
            sl_token_t* tok = next_token(ps);
            if(peek_token(ps)->type == SL_TOK_SELF) {
                return sl_intern2(ps->vm, sl_string_concat(ps->vm, tok->str, next_token(ps)->str));
            } else {
                return sl_intern2(ps->vm, tok->str);
            }
            break;
        }
        /* keywords: */
        case SL_TOK_LAST:
        case SL_TOK_NEXT:
            return sl_intern2(ps->vm, next_token(ps)->str);
        case SL_TOK_OPEN_BRACKET:
            next_token(ps);
            expect_token(ps, SL_TOK_CLOSE_BRACKET);
            return sl_intern(ps->vm, "[]");
        default:
            unexpected(ps, next_token(ps));
            SLID dummy;
            return dummy; /* never reached */
    }
}

static sl_node_base_t*
low_precedence_logical_expression(sl_parse_state_t* ps)
{
    sl_node_base_t* left = low_precedence_not_expression(ps);
    if(peek_token(ps)->type == SL_TOK_LP_AND) {
        next_token(ps);
        left = sl_make_binary_node(ps, left, low_precedence_logical_expression(ps), SL_NODE_AND);
    }
    if(peek_token(ps)->type == SL_TOK_LP_OR) {
        next_token(ps);
        left = sl_make_binary_node(ps, left, low_precedence_logical_expression(ps), SL_NODE_OR);
    }
    return left;
}

static sl_node_base_t*
mul_expression(sl_parse_state_t* ps)
{
    sl_node_base_t* left = unary_expression(ps);
    sl_node_base_t* right;
    sl_token_t* tok;
    while(peek_token(ps)->type == SL_TOK_TIMES || peek_token(ps)->type == SL_TOK_DIVIDE ||
            peek_token(ps)->type == SL_TOK_MOD) {
        tok = next_token(ps);
        right = unary_expression(ps);
        left = sl_make_send_node(ps, left, sl_intern2(ps->vm, tok->str), 1, &right, false);
    }
    return left;
}

static sl_node_base_t*
while_expression(sl_parse_state_t* ps)
{
    sl_node_base_t *condition, *body;
    sl_parse_scope_t scope;
    int until = 0;
    if(peek_token(ps)->type == SL_TOK_UNTIL) {
        next_token(ps);
        until = 1;
    } else {
        expect_token(ps, SL_TOK_WHILE);
    }
    condition = expression(ps);
    if(until) {
        condition = sl_make_unary_node(ps, condition, SL_NODE_NOT);
    }
    scope.prev = ps->scope;
    scope.flags = scope.prev->flags | SL_PF_CAN_NEXT_LAST;
    ps->scope = &scope;
    body = body_expression(ps);
    ps->scope = scope.prev;
    if(scope.flags & SL_PF_SCOPE_CLOSURE) {
        ps->scope->flags |= SL_PF_SCOPE_CLOSURE;
    }
    return sl_make_while_node(ps, condition, body);
}

int main (int argc, char **argv) {
   program_name = basename (argv[0]);
   scan_options (argc, argv);
   stack *stack = new_stack ();
   token *scanner = new_token (stdin);
   for (;;) {
      int token = scan_token (scanner);
      if (token == EOF) break;
      switch (token) {
         case NUMBER: do_push (stack, peek_token (scanner)); break;
         case '+': do_binop (stack, add_bigint); break;
         case '-': do_binop (stack, sub_bigint); break;
         case '*': do_binop (stack, mul_bigint); break;
         case 'c': do_clear (stack); break;
         case 'f': do_print_all (stack); break;
         case 'p': do_print (stack); break;
         default: unimplemented (token); break;
      }
   }
   
   do_clear(stack);
   free_stack(stack);
   free_token(scanner);
   DEBUGF ('m', "EXIT %d\n", exit_status);
   return EXIT_SUCCESS;
}

/*
 * Parse a sequence of numbers separated by the token specified in separator.
 * Exactly max numbers are expected.
 */
int
parse_numeric_aggregate(FILE *cfile, unsigned char *buf, int max, int separator,
    int base)
{
	char *val;
	int token, count;

	if (buf == NULL || max == 0)
		error("no space for numeric aggregate");

	for (count = 0; count < max; count++, buf++) {
		if (count && (peek_token(&val, cfile) == separator))
			token = next_token(&val, cfile);

		token = next_token(&val, cfile);

		if (token == TOK_NUMBER || (base == 16 && token == TOK_NUMBER_OR_NAME))
			/* XXX Need to check if conversion was successful. */
			convert_num(buf, val, base, 8);
		else
			break;
	}

	if (count < max) {
		parse_warn("numeric aggregate too short.");
		return (0);
	}

	return (1);
}

bool TokenStream::is_at_end_of_command() {
  Token t;
  if (peek_token(t)) {
    return (t.is_equal("\n") || t.is_equal(";"));
  }
  return true;
}

TOKEN type_specifier(void)
{
    TOKEN tok = peek_token();
    if(tok == NULL)
    {
        fprintf(stderr, "no type specifier found...\n");
        return NULL;
    }

    TOKEN type_spec;

    if( true == token_matches_keyword(tok, VOID) ||
            true == token_matches_keyword(tok, CHAR) ||
            true == token_matches_keyword(tok, SHORT) ||
            true == token_matches_keyword(tok, INT) ||
            true == token_matches_keyword(tok, LONG) ||
            //true == token_matches_keyword(tok, FLOAT) ||
            //true == token_matches_keyword(tok, DOUBLE) ||
            true == token_matches_keyword(tok, SIGNED) ||
            true == token_matches_keyword(tok, UNSIGNED) )
    {
        type_spec = get_token();
    }
    else
    {
        type_spec = NULL;
    }

    return type_spec;
}

TOKEN declaration(SYMBOL s)
{
    TOKEN t = NULL;
    TOKEN dec = NULL;

    s->kind = VARSYM; //if we have made it this far, i think it's safe to assume we are declaring a variable      
    declaration_specifiers(s);
    dec = init_declarator_list(s);
    t = peek_token();

    if(false == token_matches_delimiter(t, SEMICOLON))
    {
        //it turns out that we haven't found a variable declaration, so exit
        //and allow the function definition recognizer take a stab at parsing
        dec = NULL;
    }
    else
    {
        t = get_token(); //consume the SEMICOLON
        SYMBOL entry = searchins(s->namestring);
        copy_symbol(s, entry);
    }

    return dec;
}

/*----------------------------------------------------------------------*/
static ast_expression_t *parse_sum_op(parse_state_t *p, ast_expression_t *left) {
    token_t* token;

    token = peek_token(p);
    if (token == NULL) {
        return NULL;
    } else if (token->type == TK_PLUS || token->type == TK_MINUS) {
        ast_expression_t *right;
        ast_expression_t *bin_op;
        ast_expression_t *sum_op;

        next_token(p);
        right = parse_term(p);

        bin_op = ALLOC(sizeof(ast_expression_t));
        BZERO(bin_op);
        bin_op->tag = AST_EXPRESSION_BIN_OP;
        bin_op->u.bin_op.left = left;
        bin_op->u.bin_op.right = right;
        bin_op->u.bin_op.operation = token->type;

        sum_op = parse_sum_op(p, bin_op);
        return sum_op != NULL ? sum_op : bin_op;
    } else {
        return NULL;
    }
}