static int HttpGetHeaders(lua_State *luastate, int dir)
{
    if (!(LuaStateNeedProto(luastate, ALPROTO_HTTP)))
        return LuaCallbackError(luastate, "error: protocol not http");

    htp_tx_t *tx = LuaStateGetTX(luastate);
    if (tx == NULL)
        return LuaCallbackError(luastate, "internal error: no tx");

    htp_table_t *table = tx->request_headers;
    if (dir == 1)
        table = tx->response_headers;
    if (tx->request_headers == NULL)
        return LuaCallbackError(luastate, "no headers");

    lua_newtable(luastate);
    htp_header_t *h = NULL;
    size_t i = 0;
    size_t no_of_headers = htp_table_size(table);
    for (; i < no_of_headers; i++) {
        h = htp_table_get_index(table, i, NULL);
        LuaPushStringBuffer(luastate, bstr_ptr(h->name), bstr_len(h->name));
        LuaPushStringBuffer(luastate, bstr_ptr(h->value), bstr_len(h->value));
        lua_settable(luastate, -3);
    }
    return 1;
}

TEST(BstrTest, DupBin) {
    bstr *src = bstr_dup_mem("ABCDEFGHIJKL\000NOPQRSTUVWXYZ", 20);
    bstr *dst;
    dst = bstr_dup(src);

    EXPECT_EQ(bstr_len(src), bstr_len(dst));
    EXPECT_EQ(0, memcmp(bstr_ptr(src), bstr_ptr(dst), bstr_len(src)));

    bstr_free(src);
    bstr_free(dst);
}

static int HttpGetHeader(lua_State *luastate, int dir)
{
    if (!(LuaStateNeedProto(luastate, ALPROTO_HTTP)))
        return LuaCallbackError(luastate, "error: protocol not http");

    htp_tx_t *tx = LuaStateGetTX(luastate);
    if (tx == NULL)
        return LuaCallbackError(luastate, "internal error: no tx");

    const char *name = LuaGetStringArgument(luastate, 1);
    if (name == NULL)
        return LuaCallbackError(luastate, "1st argument missing, empty or wrong type");

    htp_table_t *headers = tx->request_headers;
    if (dir == 1)
        headers = tx->response_headers;
    if (headers == NULL)
        return LuaCallbackError(luastate, "tx has no headers");

    htp_header_t *h = (htp_header_t *)htp_table_get_c(headers, name);
    if (h == NULL || bstr_len(h->value) == 0)
        return LuaCallbackError(luastate, "header not found");

    return LuaPushStringBuffer(luastate,
            bstr_ptr(h->value), bstr_len(h->value));
}

/**
 * Determines protocol number from a textual representation (i.e., "HTTP/1.1"). This
 * function will only understand a properly formatted protocol information. It does
 * not try to be flexible.
 * 
 * @param[in] protocol
 * @return Protocol version or PROTOCOL_UNKNOWN.
 */
int htp_parse_protocol(bstr *protocol) {
    if (protocol == NULL) {
        return HTP_PROTOCOL_UNKNOWN;
    }
    
    // TODO This function uses a very strict approach to parsing, whereas
    //      browsers will typically be more flexible, allowing whitespace
    //      before and after the forward slash, as well as allowing leading
    //      zeroes in the numbers. We should be able to parse such malformed
    //      content correctly (but emit a warning).
    if (bstr_len(protocol) == 8) {
        unsigned char *ptr = bstr_ptr(protocol);
        if ((ptr[0] == 'H') && (ptr[1] == 'T') && (ptr[2] == 'T') && (ptr[3] == 'P')
            && (ptr[4] == '/') && (ptr[6] == '.')) {
            // Check the version numbers
            if (ptr[5] == '0') {
                if (ptr[7] == '9') {
                    return HTP_PROTOCOL_0_9;
                }
            } else if (ptr[5] == '1') {
                if (ptr[7] == '0') {
                    return HTP_PROTOCOL_1_0;
                } else if (ptr[7] == '1') {
                    return HTP_PROTOCOL_1_1;
                }
            }
        }
    }

    return HTP_PROTOCOL_UNKNOWN;
}

/**
 * Generic request line parser.
 *
 * @param connp
 * @return HTP_OK or HTP_ERROR
 */
int htp_parse_request_line_generic(htp_connp_t *connp) {
    htp_tx_t *tx = connp->in_tx;
    unsigned char *data = (unsigned char *)bstr_ptr(tx->request_line);
    size_t len = bstr_len(tx->request_line);
    size_t pos = 0;

    // The request method starts at the beginning of the
    // line and ends with the first whitespace character.
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    // No, we don't care if the method is empty.

    tx->request_method = bstr_memdup((char *)data, pos);
    if (tx->request_method == NULL) {
        return HTP_ERROR;
    }
    tx->request_method_number = htp_convert_method_to_number(tx->request_method);

    // Ignore whitespace after request method. The RFC allows
    // for only one SP, but then suggests any number of SP and HT
    // should be permitted.
    while ((pos < len) && (isspace(data[pos]))) {
        pos++;
    }

    size_t start = pos;

    // The URI ends with the first whitespace.
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    tx->request_uri = bstr_memdup((char *)data + start, pos - start);
    if (tx->request_uri == NULL) {
        return HTP_ERROR;
    }

    // Ignore whitespace after URI
    while ((pos < len) && (htp_is_space(data[pos]))) {
        pos++;
    }

    // Is there protocol information available?
    if (pos == len) {
        // No, this looks like a HTTP/0.9 request.
        tx->protocol_is_simple = 1;
        return HTP_OK;
    }

    // The protocol information spreads until the end of the line.
    tx->request_protocol = bstr_memdup((char *)data + pos, len - pos);
    if (tx->request_protocol == NULL) {
        return HTP_ERROR;
    }
    tx->request_protocol_number = htp_parse_protocol(tx->request_protocol);

    return HTP_OK;
}

static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
        const DetectEngineTransforms *transforms, Flow *_f,
        const uint8_t _flow_flags, void *txv, const int list_id)
{
    SCEnter();

    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
    if (buffer->inspect == NULL) {
        htp_tx_t *tx = (htp_tx_t *)txv;
        HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);

        if (tx_ud == NULL || tx_ud->request_uri_normalized == NULL) {
            SCLogDebug("no tx_id or uri");
            return NULL;
        }

        const uint32_t data_len = bstr_len(tx_ud->request_uri_normalized);
        const uint8_t *data = bstr_ptr(tx_ud->request_uri_normalized);

        InspectionBufferSetup(buffer, data, data_len);
        InspectionBufferApplyTransforms(buffer, transforms);
    }

    return buffer;
}

/**
 * Parses Cookie request header in v0 format.
 *
 * @param[in] connp
 * @return HTP_OK on success, HTP_ERROR on error
 */
htp_status_t htp_parse_cookies_v0(htp_connp_t *connp) {
    htp_header_t *cookie_header = htp_table_get_c(connp->in_tx->request_headers, "cookie");
    if (cookie_header == NULL) return HTP_OK;

    // Create a new table to store cookies
    connp->in_tx->request_cookies = htp_table_create(4);
    if (connp->in_tx->request_cookies == NULL) return HTP_ERROR;

    unsigned char *data = bstr_ptr(cookie_header->value);
    size_t len = bstr_len(cookie_header->value);
    size_t pos = 0;

    while (pos < len) {
        // Ignore whitespace at the beginning
        while ((pos < len) && (isspace((int)data[pos]))) pos++;
        if (pos == len) return HTP_OK;

        size_t start = pos;

        // Find the end of the cookie
        while ((pos < len) && (data[pos] != ';')) pos++;

        if (htp_parse_single_cookie_v0(connp, data + start, pos - start) != HTP_OK) {
            return HTP_ERROR;
        }

        // Go over the semicolon
        if (pos != len) pos++;
    }

    return HTP_OK;
}

/**
 * \brief Do the http_method content inspection for a signature.
 *
 * \param de_ctx  Detection engine context.
 * \param det_ctx Detection engine thread context.
 * \param s       Signature to inspect.
 * \param f       Flow.
 * \param flags   App layer flags.
 * \param state   App layer state.
 *
 * \retval 0 No match.
 * \retval 1 Match.
 */
int DetectEngineInspectHttpMethod(DetectEngineCtx *de_ctx,
                                  DetectEngineThreadCtx *det_ctx,
                                  Signature *s, Flow *f, uint8_t flags,
                                  void *alstate)
{
    SCEnter();
    int r = 0;
    HtpState *htp_state = NULL;
    htp_tx_t *tx = NULL;
    int idx;

    FLOWLOCK_RDLOCK(f);

    htp_state = (HtpState *)alstate;
    if (htp_state == NULL) {
        SCLogDebug("no HTTP state");
        goto end;
    }

    if (htp_state->connp == NULL || htp_state->connp->conn == NULL) {
        SCLogDebug("HTP state has no conn(p)");
        goto end;
    }

    idx = AppLayerTransactionGetInspectId(f);
    if (idx == -1) {
        goto end;
    }

    int size = (int)list_size(htp_state->connp->conn->transactions);
    for (; idx < size; idx++) {

        tx = list_get(htp_state->connp->conn->transactions, idx);
        if (tx == NULL || tx->request_method == NULL)
            continue;

        det_ctx->buffer_offset = 0;
        det_ctx->discontinue_matching = 0;
        det_ctx->inspection_recursion_counter = 0;

        r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HMDMATCH],
                                          f,
                                          (uint8_t *)bstr_ptr(tx->request_method),
                                          bstr_len(tx->request_method),
                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_HMD, NULL);
        //r = DoInspectHttpMethod(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HMDMATCH],
        //(uint8_t *)bstr_ptr(tx->request_method),
        //bstr_len(tx->request_method));
        if (r == 1) {
            break;
        }
    }

end:
    FLOWLOCK_UNLOCK(f);
    SCReturnInt(r);
}

/* Another special case callback. This one takes a htp_file_data_t pointer. */
int htpy_request_file_data_callback(htp_file_data_t *file_data) {
	long i;
	PyObject *res;
	PyObject *arglist;
	PyObject *data_key, *data_val;
	PyObject *filename_key, *filename_val;
	PyObject *tmpname_key, *tmpname_val;
	PyObject *dict = PyDict_New();

	if (!dict) {
		PyErr_SetString(htpy_error, "Unable to create dictionary.");
		return HTP_ERROR;
	}

	data_key = Py_BuildValue("s", "data");
	data_val = Py_BuildValue("s#", file_data->data, file_data->len);
	if (!data_key || !data_val) {
		Py_DECREF(dict);
		return HTP_ERROR;
	}
	if (PyDict_SetItem(dict, data_key, data_val) == -1) {
		Py_DECREF(dict);
		return HTP_ERROR;
	}

	if (file_data->file->filename) {
		filename_key = Py_BuildValue("s", "filename");
		filename_val = Py_BuildValue("s#", bstr_ptr(file_data->file->filename), bstr_len(file_data->file->filename));
		if (PyDict_SetItem(dict, filename_key, filename_val) == -1) {
			Py_DECREF(dict);
			return HTP_ERROR;
		}
	}

	if (file_data->file->tmpname) {
		tmpname_key = Py_BuildValue("s", "tmpname");
		tmpname_val = Py_BuildValue("s", file_data->file->tmpname);
		if (PyDict_SetItem(dict, tmpname_key, tmpname_val) == -1) {
			Py_DECREF(dict);
			return HTP_ERROR;
		}
	}

	arglist = Py_BuildValue("(O)", dict);
	if (!arglist)
		return HTP_ERROR;

	res = PyObject_CallObject(request_file_data_callback, arglist);
	Py_DECREF(arglist);
	if (PyErr_Occurred() != NULL) {
		PyErr_PrintEx(0);
		return HTP_ERROR;
	}
	i = PyInt_AsLong(res);
	Py_DECREF(res);
	return((int) i);
}

static PyObject *htpy_connp_get_response_protocol(PyObject *self, PyObject *args) {
	PyObject *ret;

	if (!((htpy_connp *) self)->connp->out_tx)
		Py_RETURN_NONE;

	if (!((htpy_connp *) self)->connp->out_tx->response_protocol)
		Py_RETURN_NONE;

	ret = Py_BuildValue("s#", bstr_ptr(((htpy_connp *) self)->connp->out_tx->response_protocol), bstr_len(((htpy_connp *) self)->connp->out_tx->response_protocol));
	return ret;
}

static int HttpGetRequestUriNormalized(lua_State *luastate)
{
    if (!(LuaStateNeedProto(luastate, ALPROTO_HTTP)))
        return LuaCallbackError(luastate, "error: protocol not http");

    htp_tx_t *tx = LuaStateGetTX(luastate);
    if (tx == NULL)
        return LuaCallbackError(luastate, "internal error: no tx");

    HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
    if (htud == NULL)
        return LuaCallbackError(luastate, "no htud in tx");

    if (htud->request_uri_normalized == NULL ||
        bstr_ptr(htud->request_uri_normalized) == NULL ||
        bstr_len(htud->request_uri_normalized) == 0)
        return LuaCallbackError(luastate, "no normalized uri");

    return LuaPushStringBuffer(luastate,
            bstr_ptr(htud->request_uri_normalized),
            bstr_len(htud->request_uri_normalized));
}

static void LogFilestoreMetaGetUri(FILE *fp, Packet *p, File *ff) {
    HtpState *htp_state = (HtpState *)p->flow->alstate;
    if (htp_state != NULL) {
        htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, ff->txid);
        if (tx != NULL && tx->request_uri_normalized != NULL) {
            PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
                    bstr_len(tx->request_uri_normalized));
            return;
        }
    }

    fprintf(fp, "<unknown>");
}

static void LogFileMetaGetHost(FILE *fp, Packet *p, File *ff) {
    HtpState *htp_state = (HtpState *)p->flow->alstate;
    if (htp_state != NULL) {
        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
        if (tx != NULL && tx->request_hostname != NULL) {
            PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(tx->request_hostname),
                           bstr_len(tx->request_hostname));
            return;
        }
    }

    fprintf(fp, "<unknown>");
}

/**
 * \brief Do the content inspection & validation for a signature
 *
 * \param de_ctx Detection engine context
 * \param det_ctx Detection engine thread context
 * \param s Signature to inspect
 * \param sm SigMatch to inspect
 * \param f Flow
 * \param flags app layer flags
 * \param state App layer state
 *
 * \retval 0 no match.
 * \retval 1 match.
 * \retval 2 Sig can't match.
 */
int DetectEngineInspectHttpResponseLine(ThreadVars *tv,
                                  DetectEngineCtx *de_ctx,
                                  DetectEngineThreadCtx *det_ctx,
                                  Signature *s, Flow *f, uint8_t flags,
                                  void *alstate,
                                  void *txv, uint64_t tx_id)
{
    htp_tx_t *tx = (htp_tx_t *)txv;

    if (tx->response_line == NULL) {
        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, txv, flags) > HTP_RESPONSE_LINE)
            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
        else
            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
    }

    det_ctx->discontinue_matching = 0;
    det_ctx->buffer_offset = 0;
    det_ctx->inspection_recursion_counter = 0;

#if 0
    PrintRawDataFp(stdout, (uint8_t *)bstr_ptr(tx->response_line),
            bstr_len(tx->response_line));
#endif

    /* run the inspection against the buffer */
    int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HTTP_RESLINEMATCH],
                                          f,
                                          bstr_ptr(tx->response_line),
                                          bstr_len(tx->response_line),
                                          0,
                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
    if (r == 1) {
        return DETECT_ENGINE_INSPECT_SIG_MATCH;
    } else {
        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
    }
}

/**
 * Parses request query string, if present.
 *
 * @param[in] connp
 * @param[in] raw_data
 * @param[in] raw_len
 * @return HTP_OK if query string was parsed, HTP_DECLINED if there was no query
 *         string, and HTP_ERROR on failure.
 */
htp_status_t htp_ch_urlencoded_callback_request_line(htp_tx_t *tx) {
    // Proceed only if there's something for us to parse.
    if ((tx->parsed_uri->query == NULL) || (bstr_len(tx->parsed_uri->query) == 0)) {
        return HTP_DECLINED;
    }

    // We have a non-zero length query string.

    tx->request_urlenp_query = htp_urlenp_create(tx);
    if (tx->request_urlenp_query == NULL) return HTP_ERROR;

    if (htp_urlenp_parse_complete(tx->request_urlenp_query, bstr_ptr(tx->parsed_uri->query),
            bstr_len(tx->parsed_uri->query)) != HTP_OK) {
        htp_urlenp_destroy(tx->request_urlenp_query);
        return HTP_ERROR;
    }

    // Add all parameters to the transaction.

    bstr *name = NULL;
    bstr *value = NULL;
    for (size_t i = 0, n = htp_table_size(tx->request_urlenp_query->params); i < n; i++) {
        value = htp_table_get_index(tx->request_urlenp_query->params, i, &name);

        htp_param_t *param = calloc(1, sizeof (htp_param_t));
        if (param == NULL) return HTP_ERROR;
        
        param->name = name;
        param->value = value;
        param->source = HTP_SOURCE_QUERY_STRING;
        param->parser_id = HTP_PARSER_URLENCODED;
        param->parser_data = NULL;

        if (htp_tx_req_add_param(tx, param) != HTP_OK) {
            free(param);
            return HTP_ERROR;
        }
    }

    // All the parameter data is now owned by the transaction, and
    // the parser table used to store it is no longer needed. The
    // line below will destroy just the table, leaving keys intact.
    htp_table_destroy_ex(tx->request_urlenp_query->params);
    tx->request_urlenp_query->params = NULL;

    htp_urlenp_destroy(tx->request_urlenp_query);
    tx->request_urlenp_query = NULL;

    return HTP_OK;
}

/*
 * XXX: Not sure I like mucking around in the transaction to get the status,
 * but I'm not sure of a better way.
 */
static PyObject *htpy_connp_get_response_status_string(PyObject *self, PyObject *args) {
	PyObject *ret;
	htp_tx_t *tx = NULL;

	tx = htp_list_get(((htpy_connp *) self)->connp->conn->transactions, htp_list_size(((htpy_connp *) self)->connp->conn->transactions) - 1);
	if (!tx) {
		PyErr_SetString(htpy_error, "Missing transaction.");
		return NULL;
	}

	ret = Py_BuildValue("s#", bstr_ptr(tx->response_status), bstr_len(tx->response_status));

	return ret;
}

static void LogFilestoreMetaGetReferer(FILE *fp, Packet *p, File *ff) {
    HtpState *htp_state = (HtpState *)p->flow->alstate;
    if (htp_state != NULL) {
        htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, ff->txid);
        if (tx != NULL) {
            table_t *headers;
            headers = tx->request_headers;
            htp_header_t *h = NULL;

            table_iterator_reset(headers);
            while (table_iterator_next(headers, (void **)&h) != NULL) {
                if (bstr_len(h->name) >= 7 &&
                        SCMemcmpLowercase((uint8_t *)"referer", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
                    PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
                        bstr_len(h->value));
                    return;
                }
            }
        }
    }

    fprintf(fp, "<unknown>");
}

/**
 * Parse query string, if available. This method is invoked after the
 * request line has been processed.
 *
 * @param connp
 */
int htp_ch_urlencoded_callback_request_line(htp_connp_t *connp) {
    // Parse query string, when available
    if ((connp->in_tx->parsed_uri->query != NULL) && (bstr_len(connp->in_tx->parsed_uri->query) > 0)) {
        connp->in_tx->request_urlenp_query = htp_urlenp_create(connp->in_tx);
        if (connp->in_tx->request_urlenp_query == NULL) {
            return HOOK_ERROR;
        }

        htp_urlenp_parse_complete(connp->in_tx->request_urlenp_query,
            (unsigned char *) bstr_ptr(connp->in_tx->parsed_uri->query),
            bstr_len(connp->in_tx->parsed_uri->query));

        // Is there a parameter processor?
        if (connp->cfg->parameter_processor == NULL) {
            // There's no parameter processor

            if (connp->cfg->internal_encoding == NULL) {
                // No transcoding; use the parser table directly
                connp->in_tx->request_params_query = connp->in_tx->request_urlenp_query->params;
                connp->in_tx->request_params_query_reused = 1;
            } else {
                // Transcode values
                connp->in_tx->request_params_query = connp->in_tx->request_urlenp_query->params;
                htp_transcode_params(connp, &connp->in_tx->request_params_query, 0);
            }
        } else {
            // We have a parameter processor defined, which
            // means we'll need to create a new table

            connp->in_tx->request_params_query =
                connp->cfg->create_table(table_size(connp->in_tx->request_urlenp_query->params));

            // Use the parameter processor on each parameter, storing
            // the results in the newly created table
            bstr *name;
            void *tvalue;
            table_iterator_reset(connp->in_tx->request_urlenp_query->params);
            while ((name = table_iterator_next(connp->in_tx->request_urlenp_query->params, & tvalue)) != NULL) {
                connp->cfg->parameter_processor(connp->in_tx->request_params_query, name, (bstr *)tvalue);
                // TODO Check return code
            }

            // Transcode as necessary
            htp_transcode_params(connp, &connp->in_tx->request_params_query, 1);
        }
    }

    return HOOK_OK;
}

int DetectEngineRunHttpUAMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
                             HtpState *htp_state, uint8_t flags)
{
    htp_tx_t *tx = NULL;
    uint32_t cnt = 0;
    int idx;

    /* we need to lock because the buffers are not actually true buffers
     * but are ones that point to a buffer given by libhtp */
    FLOWLOCK_RDLOCK(f);

    if (htp_state == NULL) {
        SCLogDebug("no HTTP state");
        goto end;
    }

    if (htp_state->connp == NULL || htp_state->connp->conn == NULL) {
        SCLogDebug("HTP state has no conn(p)");
        goto end;
    }

    idx = AppLayerTransactionGetInspectId(f);
    if (idx == -1) {
        goto end;
    }

    int size = (int)list_size(htp_state->connp->conn->transactions);
    for (; idx < size; idx++) {

        tx = list_get(htp_state->connp->conn->transactions, idx);
        if (tx == NULL)
            continue;

        htp_header_t *h = (htp_header_t *)table_getc(tx->request_headers,
                                                     "User-Agent");
        if (h == NULL) {
            SCLogDebug("HTTP user agent header not present in this request");
            continue;
        }

        cnt += HttpUAPatternSearch(det_ctx,
                                   (uint8_t *)bstr_ptr(h->value),
                                   bstr_len(h->value), flags);
    }

 end:
    FLOWLOCK_UNLOCK(f);
    return cnt;
}