static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
	int len = vma->vm_end - vma->vm_start;
	u32 key = vma->vm_pgoff << PAGE_SHIFT;
	struct c4iw_rdev *rdev;
	int ret = 0;
	struct c4iw_mm_entry *mm;
	struct c4iw_ucontext *ucontext;
	u64 addr, paddr;

	u64 va_regs_res = 0, va_udbs_res = 0;
	u64 len_regs_res = 0, len_udbs_res = 0;

	CTR3(KTR_IW_CXGBE, "%s:1 ctx %p vma %p", __func__, context, vma);

	CTR4(KTR_IW_CXGBE, "%s:1a pgoff 0x%lx key 0x%x len %d", __func__,
	    vma->vm_pgoff, key, len);

	if (vma->vm_start & (PAGE_SIZE-1)) {
		CTR3(KTR_IW_CXGBE, "%s:2 unaligned vm_start %u vma %p",
		    __func__, vma->vm_start, vma);
		return -EINVAL;
	}

	rdev = &(to_c4iw_dev(context->device)->rdev);
	ucontext = to_c4iw_ucontext(context);

	mm = remove_mmap(ucontext, key, len);
	if (!mm) {
		CTR4(KTR_IW_CXGBE, "%s:3 ucontext %p key %u len %u", __func__,
		    ucontext, key, len);
		return -EINVAL;
	}
	addr = mm->addr;
	kfree(mm);

	va_regs_res = (u64)rman_get_virtual(rdev->adap->regs_res);
	len_regs_res = (u64)rman_get_size(rdev->adap->regs_res);
	va_udbs_res = (u64)rman_get_virtual(rdev->adap->udbs_res);
	len_udbs_res = (u64)rman_get_size(rdev->adap->udbs_res);

	CTR6(KTR_IW_CXGBE,
	    "%s:4 addr %p, masync region %p:%p, udb region %p:%p", __func__,
	    addr, va_regs_res, va_regs_res+len_regs_res, va_udbs_res,
	    va_udbs_res+len_udbs_res);

	if (addr >= va_regs_res && addr < va_regs_res + len_regs_res) {
		CTR4(KTR_IW_CXGBE, "%s:5 MA_SYNC addr %p region %p, reglen %u",
		    __func__, addr, va_regs_res, len_regs_res);
		/*
		 * MA_SYNC register...
		 */
		paddr = vtophys(addr);
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		ret = io_remap_pfn_range(vma, vma->vm_start,
				paddr >> PAGE_SHIFT,
				len, vma->vm_page_prot);
	} else {

void
pmap_kenter_attr(vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
{

	CTR4(KTR_PMAP, "%s(%#x, %#x, %#x)", __func__, va, pa, ma);
	MMU_KENTER_ATTR(mmu_obj, va, pa, ma);
}

void
pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %u)", __func__, pmap, va, wired);
	MMU_CHANGE_WIRING(mmu_obj, pmap, va, wired);
}

vm_offset_t
pmap_dumpsys_map(struct pmap_md *md, vm_size_t ofs, vm_size_t *sz)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, *sz);
	return (MMU_DUMPSYS_MAP(mmu_obj, md, ofs, sz));
}

void
pmap_dumpsys_unmap(struct pmap_md *md, vm_size_t ofs, vm_offset_t va)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, va);
	return (MMU_DUMPSYS_UNMAP(mmu_obj, md, ofs, va));
}

void
pmap_unwire(pmap_t pmap, vm_offset_t start, vm_offset_t end)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, start, end);
	MMU_UNWIRE(mmu_obj, pmap, start, end);
}

void
pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
{
 
	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pm, va, sz);
	return (MMU_SYNC_ICACHE(mmu_obj, pm, va, sz));
}

static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_pdu *ip;
	struct icl_cxgbei_pdu *icp;

	M_ASSERTPKTHDR(m);

	ip = icl_cxgbei_new_pdu(M_NOWAIT);
	if (ip == NULL)
		CXGBE_UNIMPLEMENTED("PDU allocation failure");
	icp = ip_to_icp(ip);
	bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct
	    iscsi_bhs));
	icp->pdu_seq = ntohl(cpl->seq);
	icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD;

	/* This is the start of a new PDU.  There should be no old state. */
	MPASS(toep->ulpcb2 == NULL);
	toep->ulpcb2 = icp;

#if 0
	CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u",
	    __func__, tid, ntohs(cpl->len), m->m_len);
#endif

	m_freem(m);
	return (0);
}

static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_cxgbei_pdu *icp = toep->ulpcb2;

	M_ASSERTPKTHDR(m);

	/* Must already have received the header (but not the data). */
	MPASS(icp != NULL);
	MPASS(icp->pdu_flags == SBUF_ULP_FLAG_HDR_RCVD);
	MPASS(icp->ip.ip_data_mbuf == NULL);
	MPASS(icp->ip.ip_data_len == 0);

	m_adj(m, sizeof(*cpl));

	icp->pdu_flags |= SBUF_ULP_FLAG_DATA_RCVD;
	icp->ip.ip_data_mbuf = m;
	icp->ip.ip_data_len = m->m_pkthdr.len;

#if 0
	CTR4(KTR_CXGBE, "%s: tid %u, cpl->len dlen %u, m->m_len dlen %u",
	    __func__, tid, ntohs(cpl->len), m->m_len);
#endif

	return (0);
}

void
dumpsys_unmap_chunk(vm_paddr_t pa, size_t sz, void *va)
{

	CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
	return (MMU_DUMPSYS_UNMAP(mmu_obj, pa, sz, va));
}

void
pmap_zero_page_area(vm_page_t m, int off, int size)
{

	CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
	MMU_ZERO_PAGE_AREA(mmu_obj, m, off, size);
}

void *
pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
{

	CTR4(KTR_PMAP, "%s(%#x, %#x, %#x)", __func__, pa, size, attr);
	return (MMU_MAPDEV_ATTR(mmu_obj, pa, size, attr));
}

void
pmap_remove(pmap_t pmap, vm_offset_t start, vm_offset_t end)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, start, end);
	MMU_REMOVE(mmu_obj, pmap, start, end);
}

void
pmap_qenter(vm_offset_t start, vm_page_t *m, int count)
{

	CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, start, m, count);
	MMU_QENTER(mmu_obj, start, m, count);
}

vm_page_t
pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
{

	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
	return (MMU_EXTRACT_AND_HOLD(mmu_obj, pmap, va, prot));
}

/*
 * Intercept the return address from a freshly forked process that has NOT
 * been scheduled yet.
 *
 * This is needed to make kernel threads stay in kernel mode.
 */
void
cpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg)
{
	struct callframe *cf;

	CTR4(KTR_PROC, "%s called with td=%p func=%p arg=%p",
	    __func__, td, func, arg);

	cf = (struct callframe *)td->td_pcb->pcb_sp;

	cf->cf_func = (register_t)func;
	cf->cf_arg0 = (register_t)arg;
}

/*
 * Handle the return of a child process from fork1().  This function
 * is called from the MD fork_trampoline() entry point.
 */
void
fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
    struct trapframe *frame)
{
	struct proc *p;
	struct thread *td;
	struct thread *dtd;

	td = curthread;
	p = td->td_proc;
	KASSERT(p->p_state == PRS_NORMAL, ("executing process is still new"));

	CTR4(KTR_PROC, "fork_exit: new thread %p (td_sched %p, pid %d, %s)",
	    td, td_get_sched(td), p->p_pid, td->td_name);

	sched_fork_exit(td);
	/*
	* Processes normally resume in mi_switch() after being
	* cpu_switch()'ed to, but when children start up they arrive here
	* instead, so we must do much the same things as mi_switch() would.
	*/
	if ((dtd = PCPU_GET(deadthread))) {
		PCPU_SET(deadthread, NULL);
		thread_stash(dtd);
	}
	thread_unlock(td);

	/*
	 * cpu_fork_kthread_handler intercepts this function call to
	 * have this call a non-return function to stay in kernel mode.
	 * initproc has its own fork handler, but it does return.
	 */
	KASSERT(callout != NULL, ("NULL callout in fork_exit"));
	callout(arg, frame);

	/*
	 * Check if a kernel thread misbehaved and returned from its main
	 * function.
	 */
	if (p->p_flag & P_KPROC) {
		printf("Kernel thread \"%s\" (pid %d) exited prematurely.\n",
		    td->td_name, p->p_pid);
		kthread_exit();
	}
	mtx_assert(&Giant, MA_NOTOWNED);

	if (p->p_sysent->sv_schedtail != NULL)
		(p->p_sysent->sv_schedtail)(td);
	td->td_pflags &= ~TDP_FORKING;
}

void
sema_init(struct sema *sema, int value, const char *description)
{

	KASSERT((value >= 0), ("%s(): negative value\n", __func__));

	bzero(sema, sizeof(*sema));
	mtx_init(&sema->sema_mtx, description, "sema backing lock",
	    MTX_DEF | MTX_NOWITNESS | MTX_QUIET);
	cv_init(&sema->sema_cv, description);
	sema->sema_value = value;

	CTR4(KTR_LOCK, "%s(%p, %d, \"%s\")", __func__, sema, value, description);
}

static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	sq->queue = contigmalloc(sq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul,
	    4096, 0);

	if (sq->queue)
		sq->dma_addr = vtophys(sq->queue);
	else
		return -ENOMEM;
	sq->phys_addr = vtophys(sq->queue);
	pci_unmap_addr_set(sq, mapping, sq->dma_addr);
	CTR4(KTR_IW_CXGBE, "%s sq %p dma_addr %p phys_addr %p", __func__,
	    sq->queue, sq->dma_addr, sq->phys_addr);
	return 0;
}

static __inline void
call_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr)
{
	ksiginfo_t ksi;

	CTR4(KTR_TRAP, "%s: addr: %#x, sig: %d, code: %d",
	   __func__, addr, sig, code);

	/*
	 * TODO: some info would be nice to know
	 * if we are serving data or prefetch abort.
	 */

	ksiginfo_init_trap(&ksi);
	ksi.ksi_signo = sig;
	ksi.ksi_code = code;
	ksi.ksi_addr = (void *)addr;
	trapsignal(td, &ksi);
}