void
cpu_exit_wait(
	int cpu)
{
    	cpu_data_t	*cdp = cpu_datap(cpu);
	boolean_t	intrs_enabled;
	uint64_t	tsc_timeout;

	/*
	 * Wait until the CPU indicates that it has stopped.
	 * Disable interrupts while the topo lock is held -- arguably
	 * this should always be done but in this instance it can lead to
	 * a timeout if long-running interrupt were to occur here.
	 */
	intrs_enabled = ml_set_interrupts_enabled(FALSE);
	simple_lock(&x86_topo_lock);
	/* Set a generous timeout of several seconds (in TSC ticks) */
	tsc_timeout = rdtsc64() + (10ULL * 1000 * 1000 * 1000);
	while ((cdp->lcpu.state != LCPU_HALT)
	       && (cdp->lcpu.state != LCPU_OFF)
	       && !cdp->lcpu.stopped) {
	    simple_unlock(&x86_topo_lock);
	    ml_set_interrupts_enabled(intrs_enabled);
	    cpu_pause();
	    if (rdtsc64() > tsc_timeout)
		panic("cpu_exit_wait(%d) timeout", cpu);
	    ml_set_interrupts_enabled(FALSE);
	    simple_lock(&x86_topo_lock);
	}
	simple_unlock(&x86_topo_lock);
	ml_set_interrupts_enabled(intrs_enabled);
}

void
action_thread(void)
{
	register processor_t	processor;
	spl_t			s;

	thread_swappable(current_act(), FALSE);

	while (TRUE) {
		s = splsched();
		simple_lock(&action_lock);
		while ( !queue_empty(&action_queue)) {
			processor = (processor_t) queue_first(&action_queue);
			queue_remove(&action_queue, processor, processor_t,
				     processor_queue);
			simple_unlock(&action_lock);
			splx(s);

			processor_doaction(processor);

			s = splsched();
			simple_lock(&action_lock);
		}

		assert_wait((event_t) &action_queue, FALSE);
		simple_unlock(&action_lock);
		splx(s);
		counter(c_action_thread_block++);
		thread_block((void (*)(void)) 0);
	}
}

/*
 *	Routine:	lock_try_read_to_write
 *	Function:
 *		Improves a read-only lock to one with
 *		write permission.  If another reader has
 *		already requested an upgrade to a write lock,
 *		the read lock is still held upon return.
 *
 *		Returns FALSE if the upgrade *failed*.
 */
boolean_t lock_try_read_to_write(
	register lock_t	l)
{
	check_simple_locks();
	simple_lock(&l->interlock);

	if (l->thread == current_thread()) {
		/*
		 *	Recursive lock
		 */
		l->read_count--;
		l->recursion_depth++;
		simple_unlock(&l->interlock);
		return TRUE;
	}

	if (l->want_upgrade) {
		simple_unlock(&l->interlock);
		return FALSE;
	}
	l->want_upgrade = TRUE;
	l->read_count--;

	while (l->read_count != 0) {
		l->waiting = TRUE;
		thread_sleep(l,
			simple_lock_addr(l->interlock), FALSE);
		simple_lock(&l->interlock);
	}

	simple_unlock(&l->interlock);
	return TRUE;
}

/*
 *	thread_stack_daemon:
 *
 *	Perform stack allocation as required due to
 *	invoke failures.
 */
static void
thread_stack_daemon(void)
{
	thread_t		thread;

	simple_lock(&thread_stack_lock);

	while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
		simple_unlock(&thread_stack_lock);

		stack_alloc(thread);
		
		(void)splsched();
		thread_lock(thread);
		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
		thread_unlock(thread);
		(void)spllo();

		simple_lock(&thread_stack_lock);
	}

	assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
	simple_unlock(&thread_stack_lock);

	thread_block((thread_continue_t)thread_stack_daemon);
	/*NOTREACHED*/
}

/*
 * As long as db_cpu is not -1 or cpu_number(), we know that debugger
 * is active on another cpu.
 */
void
lock_db(void)
{
	int	my_cpu = cpu_number();

	for (;;) {
#if	CONSOLE_ON_MASTER
	    if (my_cpu == master_cpu) {
		db_console();
	    }
#endif /* CONSOLE_ON_MASTER */
	    if (db_cpu != -1 && db_cpu != my_cpu)
		continue;

#if	CONSOLE_ON_MASTER
	    if (my_cpu == master_cpu) {
		if (!simple_lock_try(&db_lock))
		    continue;
	    }
	    else {
		simple_lock(&db_lock);
	    }
#else /* CONSOLE_ON_MASTER */
	    simple_lock(&db_lock);
#endif /* CONSOLE_ON_MASTER */
	    if (db_cpu == -1 || db_cpu == my_cpu)
		break;
	    simple_unlock(&db_lock);
	}
}

int
boot_script_exec_cmd (void *hook, task_t task, char *path, int argc,
		      char **argv, char *strings, int stringlen)
{
  struct multiboot_module *mod = hook;

  int err;

  if (task != MACH_PORT_NULL)
    {
      thread_t thread;
      struct user_bootstrap_info info = { mod, argv, 0, };
      simple_lock_init (&info.lock);
      simple_lock (&info.lock);

      err = thread_create ((task_t)task, &thread);
      assert(err == 0);
      thread->saved.other = &info;
      thread_start (thread, user_bootstrap);
      thread_resume (thread);

      /* We need to synchronize with the new thread and block this
	 main thread until it has finished referring to our local state.  */
      while (! info.done)
	{
	  thread_sleep ((event_t) &info, simple_lock_addr(info.lock), FALSE);
	  simple_lock (&info.lock);
	}
      printf ("\n");
    }

  return 0;
}

void lock_write(
	register lock_t	l)
{
	register int	i;

	check_simple_locks();
	simple_lock(&l->interlock);

	if (l->thread == current_thread()) {
		/*
		 *	Recursive lock.
		 */
		l->recursion_depth++;
		simple_unlock(&l->interlock);
		return;
	}

	/*
	 *	Try to acquire the want_write bit.
	 */
	while (l->want_write) {
		if ((i = lock_wait_time) > 0) {
			simple_unlock(&l->interlock);
			while (--i > 0 && l->want_write)
				continue;
			simple_lock(&l->interlock);
		}

		if (l->can_sleep && l->want_write) {
			l->waiting = TRUE;
			thread_sleep(l,
				simple_lock_addr(l->interlock), FALSE);
			simple_lock(&l->interlock);
		}
	}
	l->want_write = TRUE;

	/* Wait for readers (and upgrades) to finish */

	while ((l->read_count != 0) || l->want_upgrade) {
		if ((i = lock_wait_time) > 0) {
			simple_unlock(&l->interlock);
			while (--i > 0 && (l->read_count != 0 ||
					l->want_upgrade))
				continue;
			simple_lock(&l->interlock);
		}

		if (l->can_sleep && (l->read_count != 0 || l->want_upgrade)) {
			l->waiting = TRUE;
			thread_sleep(l,
				simple_lock_addr(l->interlock), FALSE);
			simple_lock(&l->interlock);
		}
	}
	simple_unlock(&l->interlock);
}

/*
 *	Routine:	lock_read_to_write
 *	Function:
 *		Improves a read-only lock to one with
 *		write permission.  If another reader has
 *		already requested an upgrade to a write lock,
 *		no lock is held upon return.
 *
 *		Returns TRUE if the upgrade *failed*.
 */
boolean_t lock_read_to_write(
	register lock_t	l)
{
	register int	i;

	check_simple_locks();
	simple_lock(&l->interlock);

	l->read_count--;

	if (l->thread == current_thread()) {
		/*
		 *	Recursive lock.
		 */
		l->recursion_depth++;
		simple_unlock(&l->interlock);
		return(FALSE);
	}

	if (l->want_upgrade) {
		/*
		 *	Someone else has requested upgrade.
		 *	Since we've released a read lock, wake
		 *	him up.
		 */
		if (l->waiting && (l->read_count == 0)) {
			l->waiting = FALSE;
			thread_wakeup(l);
		}

		simple_unlock(&l->interlock);
		return TRUE;
	}

	l->want_upgrade = TRUE;

	while (l->read_count != 0) {
		if ((i = lock_wait_time) > 0) {
			simple_unlock(&l->interlock);
			while (--i > 0 && l->read_count != 0)
				continue;
			simple_lock(&l->interlock);
		}

		if (l->can_sleep && l->read_count != 0) {
			l->waiting = TRUE;
			thread_sleep(l,
				simple_lock_addr(l->interlock), FALSE);
			simple_lock(&l->interlock);
		}
	}

	simple_unlock(&l->interlock);
	return FALSE;
}

/*
 * dmio_ioctl:
 *
 *	Ioctl file op.
 */
static int
dmio_ioctl(struct file *fp, u_long cmd, void *data)
{
	struct dmio_state *ds = (struct dmio_state *) fp->f_data;
	int error, s;

	switch (cmd) {
	case FIONBIO:
	case FIOASYNC:
		return (0);

	case DMIO_SETFUNC:
	    {
		struct dmio_setfunc *dsf = data;
		struct dmover_session *dses;

		s = splsoftclock();
		simple_lock(&ds->ds_slock);

		if (ds->ds_session != NULL ||
		    (ds->ds_flags & DMIO_STATE_LARVAL) != 0) {
			simple_unlock(&ds->ds_slock);
			splx(s);
			return (EBUSY);
		}

		ds->ds_flags |= DMIO_STATE_LARVAL;

		simple_unlock(&ds->ds_slock);
		splx(s);

		dsf->dsf_name[DMIO_MAX_FUNCNAME - 1] = '\0';
		error = dmover_session_create(dsf->dsf_name, &dses);

		s = splsoftclock();
		simple_lock(&ds->ds_slock);

		if (error == 0) {
			dses->dses_cookie = ds;
			ds->ds_session = dses;
		}
		ds->ds_flags &= ~DMIO_STATE_LARVAL;

		simple_unlock(&ds->ds_slock);
		splx(s);
		break;
	    }

	default:
		error = ENOTTY;
	}

	return (error);
}

static void
rf_RaidIOThread(RF_ThreadArg_t arg)
{
	RF_Raid_t *raidPtr;
	RF_DiskQueueData_t *req;
	int s;

	raidPtr = (RF_Raid_t *) arg;

	s = splbio();
	simple_lock(&(raidPtr->iodone_lock));

	while (!raidPtr->shutdown_raidio) {
		/* if there is nothing to do, then snooze. */
		if (TAILQ_EMPTY(&(raidPtr->iodone)) &&
		    rf_buf_queue_check(raidPtr->raidid)) {
			ltsleep(&(raidPtr->iodone), PRIBIO, "raidiow", 0,
				&(raidPtr->iodone_lock));
		}

		/* Check for deferred parity-map-related work. */
		if (raidPtr->parity_map != NULL) {
			simple_unlock(&(raidPtr->iodone_lock));
			rf_paritymap_checkwork(raidPtr->parity_map);
			simple_lock(&(raidPtr->iodone_lock));
		}

		/* See what I/Os, if any, have arrived */
		while ((req = TAILQ_FIRST(&(raidPtr->iodone))) != NULL) {
			TAILQ_REMOVE(&(raidPtr->iodone), req, iodone_entries);
			simple_unlock(&(raidPtr->iodone_lock));
			rf_DiskIOComplete(req->queue, req, req->error);
			(req->CompleteFunc) (req->argument, req->error);
			simple_lock(&(raidPtr->iodone_lock));
		}

		/* process any pending outgoing IO */
		simple_unlock(&(raidPtr->iodone_lock));
		raidstart(raidPtr);
		simple_lock(&(raidPtr->iodone_lock));

	}

	/* Let rf_ShutdownEngine know that we're done... */
	raidPtr->shutdown_raidio = 0;
	wakeup(&(raidPtr->shutdown_raidio));

	simple_unlock(&(raidPtr->iodone_lock));
	splx(s);

	kthread_exit(0);
}

/*
 * Retry wait for output queue emptied, for write.
 * No locks may be held.
 * May run on any CPU.
 */
boolean_t char_write_done(
	register io_req_t	ior)
{
	register struct tty *tp = (struct tty *)ior->io_dev_ptr;
	register spl_t s = spltty();

	simple_lock(&tp->t_lock);
	if (tp->t_outq.c_cc > TTHIWAT(tp) ||
	    (tp->t_state & TS_CARR_ON) == 0) {

	    queue_delayed_reply(&tp->t_delayed_write, ior, char_write_done);
	    simple_unlock(&tp->t_lock);
	    splx(s);
	    return FALSE;
	}
	simple_unlock(&tp->t_lock);
	splx(s);

	if (IP_VALID(ior->io_reply_port)) {
	  (void) (*((ior->io_op & IO_INBAND) ?
		    ds_device_write_reply_inband :
		    ds_device_write_reply))(ior->io_reply_port,
					    ior->io_reply_port_type,
					    ior->io_error,
					    (int) (ior->io_total -
						   ior->io_residual));
	}
	mach_device_deallocate(ior->io_device);
	return TRUE;
}

boolean_t
lock_try_read(
	register lock_t	* l)
{
	start_data_node_t  entry = {0};
	unsigned short     trace = 0;
	pc_t		   pc;

        ETAP_STAMP(lock_event_table(l), trace, trace);
        ETAP_CREATE_ENTRY(entry, trace);

	simple_lock(&l->interlock);

	if (l->want_write || l->want_upgrade) {
		simple_unlock(&l->interlock);
                ETAP_DESTROY_ENTRY(entry);
		return(FALSE);
	}

	l->read_count++;

        ETAP_LINK_ENTRY(l, entry, trace);

	simple_unlock(&l->interlock);

        MON_ASSIGN_PC(entry->start_pc, pc, trace);
        ETAP_DURATION_TIMESTAMP(entry, trace);

	return(TRUE);
}

/*
 * Called by main() when ufs is going to be mounted as root.
 */
lfs_mountroot()
{
	extern struct vnode *rootvp;
	struct fs *fs;
	struct mount *mp;
	struct proc *p = curproc;	/* XXX */
	int error;
	
	/*
	 * Get vnodes for swapdev and rootdev.
	 */
	if ((error = bdevvp(swapdev, &swapdev_vp)) ||
	    (error = bdevvp(rootdev, &rootvp))) {
		printf("lfs_mountroot: can't setup bdevvp's");
		return (error);
	}
	if (error = vfs_rootmountalloc("lfs", "root_device", &mp))
		return (error);
	if (error = lfs_mountfs(rootvp, mp, p)) {
		mp->mnt_vfc->vfc_refcount--;
		vfs_unbusy(mp, p);
		free(mp, M_MOUNT);
		return (error);
	}
	simple_lock(&mountlist_slock);
	CIRCLEQ_INSERT_TAIL(&mountlist, mp, mnt_list);
	simple_unlock(&mountlist_slock);
	(void)lfs_statfs(mp, &mp->mnt_stat, p);
	vfs_unbusy(mp, p);
	return (0);
}

/* Updated every time a logical CPU goes offline/online */
void
commpage_update_active_cpus(void)
{
	char	    *cp;
	volatile uint8_t    *ip;
	
	/* At least 32-bit commpage must be initialized */
	if (!commPagePtr32)
		return;

	simple_lock(&commpage_active_cpus_lock);

	cp = commPagePtr32;
	cp += (_COMM_PAGE_ACTIVE_CPUS - _COMM_PAGE32_BASE_ADDRESS);
	ip = (volatile uint8_t*) cp;
	*ip = (uint8_t) processor_avail_count;
	
	cp = commPagePtr64;
	if ( cp ) {
		cp += (_COMM_PAGE_ACTIVE_CPUS - _COMM_PAGE32_START_ADDRESS);
		ip = (volatile uint8_t*) cp;
		*ip = (uint8_t) processor_avail_count;
	}

	simple_unlock(&commpage_active_cpus_lock);
}

void
himem_revert(
	hil_t		hil)
{
	hil_t		next;
	boolean_t	wakeup = FALSE;
	spl_t		ipl;

	while(hil) {
		if (hil->length) {
			bcopy((char *)phystokv(hil->low_page + hil->offset),
				(char *)phystokv(hil->high_addr),
			      hil->length);
		}
		hil->high_addr = 0;
		hil->length = 0;
		hil->offset = 0;
		next = hil->next;
		ipl = splhi();
		simple_lock(&hil_lock);
		if (!(hil->next = hil_head))
			wakeup = TRUE;
		hil_head = hil;
		simple_unlock(&hil_lock);
		splx(ipl);
		hil = next;
	}
	if (wakeup)
		thread_wakeup((event_t)&hil_head);
}

static __inline void
uvmfault_anonflush(struct vm_anon **anons, int n)
{
	int lcv;
	struct vm_page *pg;
	
	for (lcv = 0 ; lcv < n ; lcv++) {
		if (anons[lcv] == NULL)
			continue;
		simple_lock(&anons[lcv]->an_lock);
		pg = anons[lcv]->an_page;
		if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) {
			uvm_lock_pageq();
			if (pg->wire_count == 0) {
#ifdef UBC
				pmap_clear_reference(pg);
#else
				pmap_page_protect(pg, VM_PROT_NONE);
#endif
				uvm_pagedeactivate(pg);
			}
			uvm_unlock_pageq();
		}
		simple_unlock(&anons[lcv]->an_lock);
	}
}

/*
 * Handle port-death (dead reply port) for tty.
 * No locks may be held.
 * May run on any CPU.
 */
boolean_t
tty_portdeath(
	struct tty *	tp,
	ipc_port_t	port)
{
	register spl_t	spl = spltty();
	register boolean_t	result;

	simple_lock(&tp->t_lock);

	/*
	 * The queues may never have been initialized
	 */
	if (tp->t_delayed_read.next == 0) {
	    result = FALSE;
	}
	else {
	    result =
		tty_queue_clean(&tp->t_delayed_read,  port,
				tty_close_read_reply)
	     || tty_queue_clean(&tp->t_delayed_write, port,
				tty_close_write_reply)
	     || tty_queue_clean(&tp->t_delayed_open,  port,
				tty_close_open_reply);
	}
	simple_unlock(&tp->t_lock);
	splx(spl);

	return result;
}

int perfmon_release_facility(task_t task)
{
	kern_return_t retval = KERN_SUCCESS;
	task_t old_perfmon_owner = hw_perfmon_owner;
  
	simple_lock(&hw_perfmon_lock);
  
	if(task!=hw_perfmon_owner) {
		retval = KERN_NO_ACCESS;
	} else {
		if(old_perfmon_owner==kernel_task) {
			if(hw_perfmon_thread_count>0) {
#ifdef HWPERFMON_DEBUG
				kprintf("perfmon_release_facility - NOT RELEASED: kernel task is owner and has active perfmon threads\n");
#endif
				retval = KERN_NO_ACCESS;
			} else {
#ifdef HWPERFMON_DEBUG
				kprintf("perfmon_release_facility - RELEASED: kernel task was owner\n");
#endif
				hw_perfmon_owner = TASK_NULL;
				retval = KERN_SUCCESS;
			}
		} else {
#ifdef HWPERFMON_DEBUG
			kprintf("perfmon_release_facility - RELEASED: user task was owner\n");
#endif
			hw_perfmon_owner = TASK_NULL;
			retval = KERN_SUCCESS;
		}
	}

	simple_unlock(&hw_perfmon_lock);
	return retval;
}

/*
 * Retry wait for CARR_ON for open.
 * No locks may be held.
 * May run on any CPU.
 */
boolean_t char_open_done(
	io_req_t	ior)
{
	register struct tty *tp = (struct tty *)ior->io_dev_ptr;
	spl_t s = spltty();

	simple_lock(&tp->t_lock);
	if ((tp->t_state & TS_ISOPEN) == 0) {
	    queue_delayed_reply(&tp->t_delayed_open, ior, char_open_done);
	    simple_unlock(&tp->t_lock);
	    splx(s);
	    return FALSE;
	}

	tp->t_state |= TS_ISOPEN;
	tp->t_state &= ~TS_WOPEN;

	if (tp->t_mctl)
		(*tp->t_mctl)(tp, TM_RTS, DMBIS);

	simple_unlock(&tp->t_lock);
	splx(s);

	ior->io_error = D_SUCCESS;
	(void) ds_open_done(ior);
	return TRUE;
}

/*
 * sysmon_wdog_register:
 *
 *	Register a watchdog device.
 */
int
sysmon_wdog_register(struct sysmon_wdog *smw)
{
	struct sysmon_wdog *lsmw;
	int error = 0;

	simple_lock(&sysmon_wdog_list_slock);

	for (lsmw = LIST_FIRST(&sysmon_wdog_list); lsmw != NULL;
	     lsmw = LIST_NEXT(lsmw, smw_list)) {
		if (strcmp(lsmw->smw_name, smw->smw_name) == 0) {
			error = EEXIST;
			goto out;
		}
	}

	smw->smw_mode = WDOG_MODE_DISARMED;
	smw->smw_tickler = (pid_t) -1;
	smw->smw_refcnt = 0;
	sysmon_wdog_count++;
	LIST_INSERT_HEAD(&sysmon_wdog_list, smw, smw_list);

 out:
	simple_unlock(&sysmon_wdog_list_slock);
	return (error);
}