void hash_add(hash *hashtable[], bstring key, bstring value, unsigned int tablesize)
{ 
    hash *tmp = NULL;
    unsigned int thehash = makehash(key, tablesize);

    /* if there's not a value at thehash ... */
    if (hashtable[thehash]->key == NULL)
    {
        hashtable[thehash] = hash_alloc();
        hashtable[thehash]->key = bstrcpy(key);
        hashtable[thehash]->value = bstrcpy(value);
    }
    else
    {
        /* traverse through the linked list (collision resolution) */
        for (tmp = hashtable[thehash]; tmp != NULL; tmp = tmp->next)
        {
            if (tmp->next == NULL)
            {
                tmp->next = hash_alloc();
                tmp->next->key = bstrcpy(key);
                tmp->next->value = bstrcpy(value);
                break;
            }
        }
    }
}

Handler *Handler_create(bstring send_spec, bstring send_ident,
        bstring recv_spec, bstring recv_ident)
{
    debug("Creating handler %s:%s", bdata(send_spec), bdata(send_ident));

    if(!HANDLER_STACK) {
        HANDLER_STACK = Setting_get_int("limits.handler_stack", DEFAULT_HANDLER_STACK);
        log_info("MAX limits.handler_stack=%d", HANDLER_STACK);
    }

    Handler *handler = calloc(sizeof(Handler), 1);
    check_mem(handler);

    handler->send_ident = bstrcpy(send_ident);
    handler->recv_ident = bstrcpy(recv_ident);
    handler->recv_spec = bstrcpy(recv_spec);
    handler->send_spec = bstrcpy(send_spec);
    handler->running = 0;
    handler->raw = 0;
    handler->protocol = HANDLER_PROTO_JSON;

    return handler;
error:

    if(handler) free(handler);
    return NULL;
}

bKeyValues *get_key_values(const_bstring input)
{
    int i = 0;
    struct bstrList *values;
    struct bstrList *keyvalue;
    bKeyValues *keyvalues;

    values = bsplit(input, '&');
    keyvalues = malloc(sizeof(bKeyValues));
    keyvalues->entry = malloc(values->qty * sizeof(bKeyValue));
    keyvalues->qty = values->qty;

    for(i = 0; i < values->qty; i++) {
        keyvalue = bsplit(values->entry[i], '=');
        if(keyvalue->qty == 2) {
            keyvalues->entry[i].key = bstrcpy(keyvalue->entry[0]);
            keyvalues->entry[i].value = bstrcpy(keyvalue->entry[1]);
        } else {
            printf("Invalid keyvalue: %s", values->entry[i]->data);
            return NULL;
        }
        bstrListDestroy(keyvalue);
    }

    bstrListDestroy(values);

    return keyvalues;
}

Host *Host_create(bstring name, bstring matching)
{
    if(!MAX_URL_PATH || !MAX_HOST_NAME) {
        MAX_URL_PATH = Setting_get_int("limits.url_path", 256);
        MAX_HOST_NAME = Setting_get_int("limits.host_name", 256);
        log_info("MAX limits.url_path=%d, limits.host_name=%d",
                MAX_URL_PATH, MAX_HOST_NAME);
    }

    Host *host = h_calloc(sizeof(Host), 1);
    check_mem(host);

    host->name = bstrcpy(name);
    check(blength(host->name) < MAX_HOST_NAME, "Host name too long (max %d): '%s'\n", 
            MAX_HOST_NAME, bdata(name));

    host->matching = bstrcpy(matching);

    check(blength(host->matching) < MAX_HOST_NAME, "Host matching pattern too long (max %d): '%s'\n", 
            MAX_HOST_NAME, bdata(name));

    host->routes = RouteMap_create(backend_destroy_cb);
    check(host->routes, "Failed to create host route map for %s.", bdata(name));
    
    return host;

error:
    return NULL;
}

int Upload_file(Connection *conn, Handler *handler, int content_len)
{
    int rc = 0;
    int tmpfd = 0;
    bstring tmp_name = NULL;
    bstring result = NULL;

    if(UPLOAD_STORE == NULL) {
        UPLOAD_STORE = Setting_get_str("upload.temp_store", NULL);
        error_unless(UPLOAD_STORE, conn, 413, "Request entity is too large: %d, and no upload.temp_store setting for where to put the big files.", content_len);

        UPLOAD_STORE = bstrcpy(UPLOAD_STORE);
    }

    if(UPLOAD_MODE == 0) {
        bstring mode = Setting_get_str("upload.temp_store_mode", &UPLOAD_MODE_DEFAULT);
        log_info("Will set mode for upload temp store to: %s", bdata(mode));

        check(bdata(mode) != NULL, "Mode data is NULL")
        UPLOAD_MODE = strtoul((const char *)bdata(mode), NULL, 0);
        check(UPLOAD_MODE > 0, "Failed to convert upload.temp_store_mode to a number.");
        check(UPLOAD_MODE < 066666, "Invalid mode that's way too big: %s.", bdata(mode));
    }

    tmp_name = bstrcpy(UPLOAD_STORE);

    tmpfd = mkstemp((char *)tmp_name->data);
    check(tmpfd != -1, "Failed to create secure tempfile, did you end it with XXXXXX?");

    log_info("Writing tempfile %s for large upload.", bdata(tmp_name));

    rc = chmod((char *)tmp_name->data, UPLOAD_MODE);
    check(rc == 0, "Failed to chmod.");

    rc = Upload_notify(conn, handler, "start", tmp_name);
    check(rc == 0, "Failed to notify of the start of upload.");

    rc = stream_to_disk(conn->iob, content_len, tmpfd);
    check(rc == 0, "Failed to stream to disk.");

    rc = Upload_notify(conn, handler, "done", tmp_name);
    check(rc == 0, "Failed to notify the end of the upload.");

    bdestroy(result);
    bdestroy(tmp_name);
    fdclose(tmpfd);
    return 0;

error:
    if(result) bdestroy(result);
    fdclose(tmpfd);

    if(tmp_name != NULL) {
        unlink((char *)tmp_name->data);
        bdestroy(tmp_name);
    }

    return -1;
}

bstring check_known_port(uint8_t proto, uint16_t port)
{
    if (services[port] == NULL) return NULL;

    if (proto == IP_PROTO_TCP && services[port]->proto & 0x01) 
        return bstrcpy(services[port]->service_name);
    if (proto == IP_PROTO_UDP && services[port]->proto & 0x02) 
        return bstrcpy(services[port]->service_name);

    return NULL;
}

Server *Server_create(bstring uuid, bstring default_host,
        bstring bind_addr, int port, bstring chroot, bstring access_log,
        bstring error_log, bstring pid_file, bstring control_port, int use_ssl)
{
    Server *srv = NULL;
    int rc = 0;

    srv = h_calloc(sizeof(Server), 1);
    check_mem(srv);

    srv->hosts = RouteMap_create(host_destroy_cb);
    check(srv->hosts != NULL, "Failed to create host RouteMap.");

    srv->handlers = darray_create(sizeof(Handler), 20);
    check_mem(srv->handlers);

    check(port > 0, "Invalid port given, must be > 0: %d", port);
    srv->port = port;
    srv->listen_fd = 0;

    srv->bind_addr = bstrcpy(bind_addr); check_mem(srv->bind_addr);
    srv->uuid = bstrcpy(uuid); check_mem(srv->uuid);

    // TODO: once mbedtls supports opening urandom early and keeping it open,
    //   put the rng initialization back here (before chroot)
    //if(use_ssl) {
    //    rc = Server_init_rng(srv);
    //    check(rc == 0, "Failed to initialize rng for server %s", bdata(uuid));
    //}

    if(blength(chroot) > 0) {
        srv->chroot = bstrcpy(chroot); check_mem(srv->chroot);
    } else {
        srv->chroot = NULL;
    }
    srv->access_log = bstrcpy(access_log); check_mem(srv->access_log);
    srv->error_log = bstrcpy(error_log); check_mem(srv->error_log);
    srv->pid_file = bstrcpy(pid_file); check_mem(srv->pid_file);
    if(blength(control_port) > 0) {
        srv->control_port = bstrcpy(control_port); check_mem(srv->control_port);
    } else {
        srv->control_port = NULL;
    }
    srv->default_hostname = bstrcpy(default_host);
    srv->use_ssl = use_ssl;
    srv->created_on = time(NULL);

    if(srv->use_ssl) {
        rc = Server_init_ssl(srv);
        check(rc == 0, "Failed to initialize ssl for server %s", bdata(uuid));
    }

    return srv;

error:
    Server_destroy(srv);
    return NULL;
}

int test1 (void) {
struct tagbstring t = bsStatic ("Hello world");
bstring b, c, d;
int ret = 0;

	printf ("TEST: bTail and bHead functions.\n");
	b = bTail (&t, 5);
	c = bHead (&t, 5);
	ret += 0 >= biseqcstr (b, "world");
	ret += 0 >= biseqcstr (c, "Hello");
	bdestroy (b);
	bdestroy (c);

	b = bTail (&t, 0);
	c = bHead (&t, 0);
	ret += 0 >= biseqcstr (b, "");
	ret += 0 >= biseqcstr (c, "");
	bdestroy (b);
	bdestroy (c);

	d = bstrcpy (&t);
	b = bTail (d, 5);
	c = bHead (d, 5);
	ret += 0 >= biseqcstr (b, "world");
	ret += 0 >= biseqcstr (c, "Hello");
	bdestroy (b);
	bdestroy (c);
	bdestroy (d);

	printf ("\t# failures: %d\n", ret);

	return ret;
}

int gen_keys(DArray *keys, int num_keys) {
	int i = 0;
	FILE *urand = fopen("/dev/urandom", "r");
	check(urand != NULL, "Failed to open /dev/urandom");

	struct bStream *stream = bsopen((bNread)fread, urand);
	check(stream != NULL, "Failed to open /dev/urandom");

	bstring key = bfromcstr("");
	int rc = 0;

	// FNVla histogram
	for(i = 0; i < num_keys; i++) {
		rc = bsread(key, stream, BUFFER_LEN);
		check(rc >= 0, "Failed to read from /dev/urandom.");

		DArray_push(keys, bstrcpy(key));
	}

	bsclose(stream);
	fclose(urand);
	return 0;

error:
	return -1;
}

int Upload_notify(Connection *conn, Handler *handler, const char *stage, bstring tmp_name)
{
    bstring key = bformat("x-mongrel2-upload-%s", stage);
    Request_set(conn->req, key, bstrcpy(tmp_name), 1);

    return Connection_send_to_handler(conn, handler, "", 0, NULL);
}

///
/// Stores the application path in a global context.
///
/// Stores the application path (arg0) in a global context so that
/// libraries can retrieve it later using osutil_getarg0.  All applications
/// using a DCPU-16 Toolchain library should invoke this method after
/// parsing their arguments.  If this value is not set, some libraries
/// may be unable to resolve their required runtime components.
///
/// @param arg0 The string containing argument 0.
///
void osutil_setarg0(freed_bstring arg0)
{
    if (osutil_arg0 != NULL)
        bdestroy(osutil_arg0);
    osutil_arg0 = bstrcpy(arg0.ref);
    bautodestroy(arg0);
}

// Adds a dependency onto the list of dependencies.
//
// dependency   - The name of the dependency.
// dependencies - A pointer to an array of dependencies.
// count        - A pointer to where the number of dependencies is stored.
//
// Returns 0 if successful, otherwise returns -1.
int qip_ast_node_add_dependency(bstring dependency, bstring **dependencies, uint32_t *count)
{
    check(dependencies != NULL, "Dependency array pointer required");
    check(count != NULL, "Dependency count pointer required");
    
    // If dependency is blank or null or it is a built-in type then ignore it.
    if(dependency == NULL || biseqcstr(dependency, "") || qip_is_builtin_type_name(dependency)) {
        return 0;
    }
    
    // If dependency exists then exit.
    uint32_t i;
    for(i=0; i<*count; i++) {
        if(biseq(dependency, (*dependencies)[i])) {
            return 0;
        }
    }
    
    // Increment the count and append.
    (*count)++;
    *dependencies = realloc(*dependencies, sizeof(bstring) * (*count));
    check_mem(*dependencies);
    (*dependencies)[(*count)-1] = bstrcpy(dependency);
    check_mem((*dependencies)[(*count)-1]);
    
    return 0;

error:
    qip_ast_node_dependencies_free(dependencies, count);
    return -1;
}

alder_thread_readwriter_t * alder_thread_readwriter_create(bstring bfnin,
                                                           struct bstrList *bfnout,
                                                           int size_inbuf,
                                                           int size_outbuf)
{
    alder_thread_readwriter_t *o = malloc(sizeof(*o));
    ALDER_RETURN_NULL_IF_NULL(o);
    
    o->bfnin = bstrcpy(bfnin);
    o->bfnout = bstrVectorCopy(bfnout);
    
    o->begin_inbuf = 0;
    o->end_inbuf = 0;
    o->end_outbuf = malloc(sizeof(*o->end_outbuf)*o->bfnout->qty);
    o->size_inbuf = size_inbuf;
    o->size_outbuf = size_outbuf;
    o->inbuf = malloc(sizeof(*o->inbuf) * size_inbuf);
    o->outbuf = malloc(sizeof(*o->outbuf) * o->bfnout->qty);
    for (int j = 0; j < o->bfnout->qty; j++) {
        o->outbuf[j] = malloc(sizeof(*o->outbuf[j]) * size_outbuf);
    }
    
    o->fpi = fopen(bdata(o->bfnin), "r");
    if (o->fpi == NULL) {
        perror("cannot open the input file.");
    }
    o->fpo = malloc(sizeof(*o->fpo) * o->bfnout->qty);
    for (int j = 0; j < o->bfnout->qty; j++) {
        o->fpo[j] = fopen(bdata(o->bfnout->entry[j]), "w");
        if (o->fpo[j] == NULL) {
            perror("cannot open an output file.");
        }
    }
    return o;
}

IDENT_MPTR_RAW * BetterString_$_splits_$_String(IDENT_MPTR_RAW *  b_, IDENT_MPTR_RAW *  s_, IDENT_MPTR_RAW * $_mptr_in)
{
    const_bstring b = b_->obj;
    String * s = s_->obj;
    bstring sBstring = bfromcstr(s->buffer);
    struct bstrList * result = bsplits(b,sBstring);

    _$_VARIABLE(_$_temp_vector);
    _$_VARIABLE(_$_temp_vector_item);
    _$_VARIABLE(_$_temp_betterstring);

    //printf("Parts %d ",result->qty);
    Vector_$_Vector_$_int(result->qty, _$_temp_vector, false, BetterString);
    /*
    Vector_$_putObjectAtIndex_$_Generic_$$_$_Generic_$$(_$v$_arr, _$v$_idx, _$v$_primIdx, _$v$_elem, _$v$_primElem,\
                                                    _$v$_mptr, _$v$_primRet, _$v$_typeRet)    */
    int i;

    for (i=0;i<result->qty;i++) {
        BetterString_$_BetterString_$_BetterString_$_(bstrcpy(result->entry[i]),_$_temp_betterstring);
        Vector_$_putObjectAtIndex_$_Generic_$$_$_Generic_$$(_$_temp_vector, i, true,
                                                            _$_temp_betterstring,
                                                            false, _$_temp_vector_item, false, _$_mptr);
    }
    _$_mptr_prepare(_$_temp_vector,$_mptr_in);
    bdestroy(sBstring);
    bstrListDestroy(result);
    return $_mptr_in;
}

int parse_streams(Workgroup* group, const_bstring str, int numberOfStreams)
{
    struct bstrList* tokens;
    struct bstrList* subtokens;
    tokens = bsplit(str,',');

    if (tokens->qty < numberOfStreams)
    {
        fprintf(stderr, "Error: Testcase requires at least %d streams\n", numberOfStreams);
        bstrListDestroy(tokens);
        return -1;
    }

    group->streams = (Stream*) malloc(numberOfStreams * sizeof(Stream));
    if (group->streams == NULL)
    {
        bstrListDestroy(tokens);
        return -1;
    }
    for (int i=0; i<numberOfStreams; i++)
    {
        subtokens = bsplit(tokens->entry[i],':');
        if (subtokens->qty >= 2)
        {
            int index = str2int(bdata(subtokens->entry[0]));
            if ((index < 0) && (index >= numberOfStreams))
            {
                free(group->streams);
                bstrListDestroy(subtokens);
                bstrListDestroy(tokens);
                return -1;
            }
            group->streams[index].domain = bstrcpy(subtokens->entry[1]);
            group->streams[index].offset = 0;
            if (subtokens->qty == 3)
            {
                group->streams[index].offset = str2int(bdata(subtokens->entry[2]));
                if (group->streams[index].offset < 0)
                {
                free(group->streams);
                bstrListDestroy(subtokens);
                bstrListDestroy(tokens);
                return -1;
                }
            }
        }
        else
        {
            fprintf(stderr, "Error in parsing stream definition %s\n", bdata(tokens->entry[i]));
            bstrListDestroy(subtokens);
            bstrListDestroy(tokens);
            free(group->streams);
            return -1;
        }
        bstrListDestroy(subtokens);
    }

    bstrListDestroy(tokens);
    return 0;
}

int Host_add_backend(Host *host, bstring path, BackendType type, void *target)
{
    debug("ADDING ROUTE TO HOST %p: %s", host, bdata(path));
    Backend *backend = calloc(sizeof(Backend), 1);
    check_mem(backend);

    backend->type = type;

    if(type == BACKEND_HANDLER) {
        backend->target.handler = target;
    } else if(type == BACKEND_PROXY) {
        backend->target.proxy = target;
    } else if(type == BACKEND_DIR) {
        backend->target.dir = target;
    } else {
        sentinel("Invalid proxy type given: %d", type);
    }

    int rc = RouteMap_insert(host->routes, bstrcpy(path), backend);
    check(rc == 0, "Failed to insert %s into host %s route map.",
            bdata(path), bdata(host->name));

    return 0;
    
error:
    return -1;
}

TSTree *add_route_data(TSTree *routes, bstring line)
{
    struct bstrList *data = bsplit(line, ' ');
    // eg. /hello && Hello

    check(data->qty == 2, "Line '%s' does not have 2 columns",
          bdata(line));


    // insert to routes, root is the first, the 2 and 3 is key anf length of key and value
    routes = TSTree_insert(routes,
                           bdata(data->entry[0]), blength(data->entry[0]),
                           bstrcpy(data->entry[1]));

    void *dl = load_dl(data->entry[1]);

    Handler *handler = Handler_create(bdata(data->entry[0]), dl);



    bstrListDestroy(data);

    return routes;

error:
    return NULL;
}

// Deletes a table to the server. This function is synchronous and does not use
// a worker.
//
// server - The server.
// header - The message header.
// table  - The table the message is working against
// input  - The input file stream.
// output - The output file stream.
//
// Returns 0 if successful, otherwise returns -1.
int sky_delete_table_message_process(sky_server *server,
                                     sky_message_header *header,
                                     sky_table *_table, FILE *input, FILE *output)
{
    int rc = 0;
    size_t sz;
    sky_delete_table_message *message = NULL;
    sky_table *table = NULL;
    bstring path = NULL;
    check(server != NULL, "Server required");
    check(header != NULL, "Message header required");
    check(input != NULL, "Input stream required");
    check(output != NULL, "Output stream required");
    (void)_table;
    
    struct tagbstring status_str = bsStatic("status");
    struct tagbstring ok_str = bsStatic("ok");

    // Parse message.
    message = sky_delete_table_message_create(); check_mem(message);
    rc = sky_delete_table_message_unpack(message, input);
    check(rc == 0, "Unable to parse 'delete_table' message");

    // Retrieve table reference from server.
    rc = sky_server_get_table(server, message->name, &table);
    check(rc == 0, "Unable to find table: %s", bdata(message->name));
    check(table != NULL, "Table does not exist: %s", bdata(message->name));

    // Detach table first.
    path = bstrcpy(table->path); check_mem(path);
    rc = sky_server_close_table(server, table);
    check(rc == 0, "Unable to close table before deletion");

    // If the table exists then delete it.
    if(sky_file_exists(path)) {
        rc = sky_file_rm_r(path);
        check(rc == 0, "Unable to delete table: %s", bdata(path));
    }
    
    // Return.
    //   {status:"OK"}
    minipack_fwrite_map(output, 1, &sz);
    check(sz > 0, "Unable to write output");
    check(sky_minipack_fwrite_bstring(output, &status_str) == 0, "Unable to write status key");
    check(sky_minipack_fwrite_bstring(output, &ok_str) == 0, "Unable to write status value");

    fclose(input);
    fclose(output);
    bdestroy(path);
    sky_delete_table_message_free(message);
    
    return 0;

error:
    if(input) fclose(input);
    if(output) fclose(output);
    bdestroy(path);
    sky_delete_table_message_free(message);
    return -1;
}

Dir *Dir_create(const char *base, const char *prefix, const char *index_file, const char *default_ctype)
{
    Dir *dir = calloc(sizeof(Dir), 1);
    check_mem(dir);

    dir->base = bfromcstr(base);
    check(blength(dir->base) < MAX_DIR_PATH, "Base directory is too long, must be less than %d", MAX_DIR_PATH);

    // dir can come from the routing table so it could have a pattern in it, strip that off
    bstring pattern = bfromcstr(prefix);
    int first_paren = bstrchr(pattern, '(');
    dir->prefix = first_paren >= 0 ? bHead(pattern, first_paren) : bstrcpy(pattern);
    bdestroy(pattern);

    check(blength(dir->prefix) < MAX_DIR_PATH, "Prefix is too long, must be less than %d", MAX_DIR_PATH);

    check(bchar(dir->prefix, 0) == '/' && bchar(dir->prefix, blength(dir->prefix)-1) == '/',
                "Dir route prefix (%s) must start with / and end with / or else you break the internet.", prefix);

    dir->index_file = bfromcstr(index_file);
    dir->default_ctype = bfromcstr(default_ctype);

    dir->fr_cache = Cache_create(FR_CACHE_SIZE, filerecord_cache_lookup,
                                 filerecord_cache_evict);
    check(dir->fr_cache, "Failed to create FileRecord cache");

    return dir;

error:
    if(dir)
        free(dir);

    return NULL;
}

int gen_keys(DArray * keys, int num_keys)
{
    int i = 0;
    FILE *urand = fopen("/dev/urandom", "r");
    check(urand != NULL, "Failed to open /dev/urandom");
    int result = -1; // default to failure condition
    int rc = 0;

    struct bStream *stream = bsopen((bNread) fread, urand);
    check(stream != NULL, "Failed to open /dev/urandom");

    bstring key = bfromcstr("");

    // FNV1a histogram
    for (i = 0; i < num_keys; i++) {
        rc = bsread(key, stream, BUFFER_LEN);
        check(rc >= 0, "Failed to read from /dev/urandom.");

        DArray_push(keys, bstrcpy(key));
    }

    result = 0; // all good

error: // fallthrough
    if(stream) bsclose(stream);
    if(urand) fclose(urand);
    if(key) bdestroy(key);
    return result;
}