void
vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,
    vm_offset_t end)
{
	struct vnode *vp;
	vm_ooffset_t old_wm;

	VM_OBJECT_WLOCK(object);
	if (object->type != OBJT_VNODE) {
		VM_OBJECT_WUNLOCK(object);
		return;
	}
	old_wm = object->un_pager.vnp.writemappings;
	object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;
	vp = object->handle;
	if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {
		ASSERT_VOP_ELOCKED(vp, "v_writecount inc");
		VOP_ADD_WRITECOUNT(vp, 1);
		CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d",
		    __func__, vp, vp->v_writecount);
	} else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {
		ASSERT_VOP_ELOCKED(vp, "v_writecount dec");
		VOP_ADD_WRITECOUNT(vp, -1);
		CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
		    __func__, vp, vp->v_writecount);
	}
	VM_OBJECT_WUNLOCK(object);
}

static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
	int len = vma->vm_end - vma->vm_start;
	u32 key = vma->vm_pgoff << PAGE_SHIFT;
	struct c4iw_rdev *rdev;
	int ret = 0;
	struct c4iw_mm_entry *mm;
	struct c4iw_ucontext *ucontext;
	u64 addr, paddr;

	u64 va_regs_res = 0, va_udbs_res = 0;
	u64 len_regs_res = 0, len_udbs_res = 0;

	CTR3(KTR_IW_CXGBE, "%s:1 ctx %p vma %p", __func__, context, vma);

	CTR4(KTR_IW_CXGBE, "%s:1a pgoff 0x%lx key 0x%x len %d", __func__,
	    vma->vm_pgoff, key, len);

	if (vma->vm_start & (PAGE_SIZE-1)) {
		CTR3(KTR_IW_CXGBE, "%s:2 unaligned vm_start %u vma %p",
		    __func__, vma->vm_start, vma);
		return -EINVAL;
	}

	rdev = &(to_c4iw_dev(context->device)->rdev);
	ucontext = to_c4iw_ucontext(context);

	mm = remove_mmap(ucontext, key, len);
	if (!mm) {
		CTR4(KTR_IW_CXGBE, "%s:3 ucontext %p key %u len %u", __func__,
		    ucontext, key, len);
		return -EINVAL;
	}
	addr = mm->addr;
	kfree(mm);

	va_regs_res = (u64)rman_get_virtual(rdev->adap->regs_res);
	len_regs_res = (u64)rman_get_size(rdev->adap->regs_res);
	va_udbs_res = (u64)rman_get_virtual(rdev->adap->udbs_res);
	len_udbs_res = (u64)rman_get_size(rdev->adap->udbs_res);

	CTR6(KTR_IW_CXGBE,
	    "%s:4 addr %p, masync region %p:%p, udb region %p:%p", __func__,
	    addr, va_regs_res, va_regs_res+len_regs_res, va_udbs_res,
	    va_udbs_res+len_udbs_res);

	if (addr >= va_regs_res && addr < va_regs_res + len_regs_res) {
		CTR4(KTR_IW_CXGBE, "%s:5 MA_SYNC addr %p region %p, reglen %u",
		    __func__, addr, va_regs_res, len_regs_res);
		/*
		 * MA_SYNC register...
		 */
		paddr = vtophys(addr);
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		ret = io_remap_pfn_range(vma, vma->vm_start,
				paddr >> PAGE_SHIFT,
				len, vma->vm_page_prot);
	} else {

static unsigned int
cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv,
			struct ccb_scsiio *scmd, unsigned int *itt)
{
	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
	int xferlen = scmd->dxfer_len;
	struct cxgbei_task_data *tdata = NULL;
	struct cxgbei_sgl *sge = NULL;
	struct toepcb *toep = icc->toep;
	struct adapter *sc = td_adapter(toep->td);
	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
	int err = -1;

	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);

	tdata = (struct cxgbei_task_data *)*prv;
	if (xferlen == 0 || tdata == NULL)
		goto out;
	if (xferlen < DDP_THRESHOLD)
		goto out;

	if ((scmd->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
		tdata->nsge = cxgbei_map_sg(tdata->sgl, scmd);
		if (tdata->nsge == 0) {
			CTR1(KTR_CXGBE, "%s: map_sg failed", __func__);
			return 0;
		}
		sge = tdata->sgl;

		tdata->sc_ddp_tag = *itt;

		CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x",
				__func__, *itt, tdata->sc_ddp_tag);
		if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format,
							tdata->sc_ddp_tag)) {
			err = t4_sk_ddp_tag_reserve(ci, icc, scmd->dxfer_len,
			    sge, tdata->nsge, &tdata->sc_ddp_tag);
		} else {
			CTR3(KTR_CXGBE,
				"%s: itt:0x%x sc_ddp_tag:0x%x not usable",
				__func__, *itt, tdata->sc_ddp_tag);
		}
	}
out:
	if (err < 0)
		tdata->sc_ddp_tag =
			cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt);

	return tdata->sc_ddp_tag;
}

void cxio_dump_tpt(struct cxio_rdev *rdev, uint32_t stag)
{
	struct ch_mem_range *m;
	u64 *data;
	int rc;
	int size = 32;

	m = kmalloc(sizeof(*m) + size, M_NOWAIT);
	if (!m) {
		CTR1(KTR_IW_CXGB, "%s couldn't allocate memory.", __FUNCTION__);
		return;
	}
	m->mem_id = MEM_PMRX;
	m->addr = (stag>>8) * 32 + rdev->rnic_info.tpt_base;
	m->len = size;
	CTR3(KTR_IW_CXGB, "%s TPT addr 0x%x len %d", __FUNCTION__, m->addr, m->len);
	rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
	if (rc) {
		CTR2(KTR_IW_CXGB, "%s toectl returned error %d", __FUNCTION__, rc);
		free(m, M_DEVBUF);
		return;
	}

	data = (u64 *)m->buf;
	while (size > 0) {
		CTR2(KTR_IW_CXGB, "TPT %08x: %016llx", m->addr, (unsigned long long) *data);
		size -= 8;
		data++;
		m->addr += 8;
	}
	free(m, M_DEVBUF);
}

void
pmap_unmapdev(vm_offset_t va, vm_size_t size)
{

	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size);
	MMU_UNMAPDEV(mmu_obj, va, size);
}

void
pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
{

	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
	return (MMU_PAGE_SET_MEMATTR(mmu_obj, m, ma));
}

void *
pmap_mapdev(vm_paddr_t pa, vm_size_t size)
{

	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
	return (MMU_MAPDEV(mmu_obj, pa, size));
}

int
pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
{

	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
	return (MMU_MINCORE(mmu_obj, pmap, addr, locked_pa));
}

void
pmap_qremove(vm_offset_t start, int count)
{

	CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, start, count);
	MMU_QREMOVE(mmu_obj, start, count);
}

boolean_t
pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
{

	CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
	return (MMU_PAGE_EXISTS_QUICK(mmu_obj, pmap, m));
}

boolean_t
pmap_is_prefaultable(pmap_t pmap, vm_offset_t va)
{

	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, va);
	return (MMU_IS_PREFAULTABLE(mmu_obj, pmap, va));
}

vm_paddr_t
pmap_extract(pmap_t pmap, vm_offset_t va)
{

	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, va);
	return (MMU_EXTRACT(mmu_obj, pmap, va));
}

void
pmap_copy_page(vm_page_t src, vm_page_t dst)
{

	CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
	MMU_COPY_PAGE(mmu_obj, src, dst);
}

int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
					struct iwch_mr *mhp,
					int shift,
					__be64 *page_list)
{
	u32 stag;
	u32 mmid;


	if (cxio_register_phys_mem(&rhp->rdev,
				   &stag, mhp->attr.pdid,
				   mhp->attr.perms,
				   mhp->attr.zbva,
				   mhp->attr.va_fbo,
				   mhp->attr.len,
				   shift-12,
				   page_list,
				   &mhp->attr.pbl_size, &mhp->attr.pbl_addr))
		return (-ENOMEM);
	mhp->attr.state = 1;
	mhp->attr.stag = stag;
	mmid = stag >> 8;
	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
	insert_handle(rhp, &rhp->mmidr, mhp, mmid);
	CTR3(KTR_IW_CXGB, "%s mmid 0x%x mhp %p", __FUNCTION__, mmid, mhp);
	return 0;
}

boolean_t
pmap_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
{

	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
	return (MMU_DEV_DIRECT_MAPPED(mmu_obj, pa, size));
}

void
pmap_kenter(vm_offset_t va, vm_paddr_t pa)
{

	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, pa);
	MMU_KENTER(mmu_obj, va, pa);
}

/*
 * Active open failed.
 */
static int
do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
    struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	const struct cpl_act_establish *cpl = (const void *)(rss + 1);
	unsigned int tid = GET_TID(cpl);
	unsigned int atid = G_TID_TID(ntohl(cpl->tos_atid));
	struct toepcb *toep = lookup_atid(sc, atid);
	struct inpcb *inp = toep->inp;

	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
	KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));

	CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid);
	free_atid(sc, atid);

	INP_WLOCK(inp);
	toep->tid = tid;
	insert_tid(sc, tid, toep);
	if (inp->inp_flags & INP_DROPPED) {

		/* socket closed by the kernel before hw told us it connected */

		send_flowc_wr(toep, NULL);
		send_reset(sc, toep, be32toh(cpl->snd_isn));
		goto done;
	}

	make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
done:
	INP_WUNLOCK(inp);
	return (0);
}

static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_cxgbei_pdu *icp = toep->ulpcb2;

	M_ASSERTPKTHDR(m);
	MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));

	/* Must already have received the header (but not the data). */
	MPASS(icp != NULL);
	MPASS(icp->icp_flags == ICPF_RX_HDR);
	MPASS(icp->ip.ip_data_mbuf == NULL);


	m_adj(m, sizeof(*cpl));
	MPASS(icp->ip.ip_data_len == m->m_pkthdr.len);

	icp->icp_flags |= ICPF_RX_FLBUF;
	icp->ip.ip_data_mbuf = m;
	counter_u64_add(ci->fl_pdus, 1);
	counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);

#if 0
	CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
	    be16toh(cpl->len));
#endif

	return (0);
}

void cxio_dump_tcb(struct cxio_rdev *rdev, uint32_t hwtid)
{
	struct ch_mem_range *m;
	int size = TCB_SIZE;
	uint32_t *data;
	int rc;

	m = kmalloc(sizeof(*m) + size, M_NOWAIT);
	if (!m) {
		CTR1(KTR_IW_CXGB, "%s couldn't allocate memory.", __FUNCTION__);
		return;
	}
	m->mem_id = MEM_CM;
	m->addr = hwtid * size;
	m->len = size;
	CTR3(KTR_IW_CXGB, "%s TCB %d len %d", __FUNCTION__, m->addr, m->len);
	rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
	if (rc) {
		CTR2(KTR_IW_CXGB, "%s toectl returned error %d", __FUNCTION__, rc);
		free(m, M_DEVBUF);
		return;
	}

	data = (uint32_t *)m->buf;
	while (size > 0) {
		printf("%2u: %08x %08x %08x %08x %08x %08x %08x %08x\n",
			m->addr,
			*(data+2), *(data+3), *(data),*(data+1),
			*(data+6), *(data+7), *(data+4), *(data+5));
		size -= 32;
		data += 8;
		m->addr += 32;
	}
	free(m, M_DEVBUF);
}

int
freebsd32_sigreturn(struct thread *td, struct freebsd32_sigreturn_args *uap)
{
	ucontext32_t uc;
	int error;

	CTR2(KTR_SIG, "sigreturn: td=%p ucp=%p", td, uap->sigcntxp);

	if (copyin(uap->sigcntxp, &uc, sizeof(uc)) != 0) {
		CTR1(KTR_SIG, "sigreturn: efault td=%p", td);
		return (EFAULT);
	}

	error = set_mcontext32(td, &uc.uc_mcontext);
	if (error != 0)
		return (error);

	kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);

#if 0
	CTR3(KTR_SIG, "sigreturn: return td=%p pc=%#x sp=%#x",
	     td, uc.uc_mcontext.mc_srr0, uc.uc_mcontext.mc_gpr[1]);
#endif

	return (EJUSTRETURN);
}