Sint
erts_complete_off_heap_message_queue_change(Process *c_p)
{
    int reds = 1;

    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
    ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
    ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);

    /*
     * This job was first initiated when the process changed to off heap
     * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ
     * has been set. However, the management state might have been changed
     * again (multiple times) since then. Check users last requested state
     * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state
     * consistent with that.
     */

    if (!(c_p->flags & F_OFF_HEAP_MSGQ))
	erts_smp_atomic32_read_band_nob(&c_p->state,
					~ERTS_PSFLG_OFF_HEAP_MSGQ);
    else {
	reds += 2;
	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
	reds += erts_move_messages_off_heap(c_p);
    }
    c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG;
    return reds;
}

static ERTS_INLINE void
notify_new_message(Process *receiver)
{
    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
                       & erts_proc_lc_my_proc_locks(receiver));

    switch (receiver->status) {
    case P_GARBING:
        switch (receiver->gcstatus) {
        case P_SUSPENDED:
            goto suspended;
        case P_WAITING:
            goto waiting;
        default:
            break;
        }
        break;
    case P_SUSPENDED:
suspended:
        receiver->rstatus = P_RUNABLE;
        break;
    case P_WAITING:
waiting:
        erts_add_to_runq(receiver);
        break;
    default:
        break;
    }
}

int 
erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
		      Eterm tracer_pid) {
    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
    return set_break(mfa, specified, match_spec,
		     (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
}

int 
erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
		      Eterm tracer_pid) {
    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
    return set_break(mfa, specified, match_spec,
		     (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
}

static ERTS_INLINE Port *
pop_port(ErtsRunQueue *runq)
{
    Port *pp = runq->ports.start;
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
    if (!pp) {
	ASSERT(!runq->ports.end);
    }
    else {
	runq->ports.start = runq->ports.start->sched.next;
	if (runq->ports.start)
	    runq->ports.start->sched.prev = NULL;
	else {
	    ASSERT(runq->ports.end == pp);
	    runq->ports.end = NULL;
	}
	ASSERT(runq->ports.info.len > 0);
	runq->ports.info.len--;
	ASSERT(runq->len > 0);
	runq->len--;
    }

    ASSERT(runq->ports.start || !runq->ports.end);
    ASSERT(runq->ports.end || !runq->ports.start);
    return pp;
}

int 
erts_clear_module_break(Module *modp) {
    BeamCodeHeader* code_hdr;
    Uint n;
    Uint i;

    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
    ASSERT(modp);
    code_hdr = modp->curr.code_hdr;
    if (!code_hdr) {
	return 0;
    }
    n = (Uint)(UWord) code_hdr->num_functions;
    for (i = 0; i < n; ++i) {
	ErtsCodeInfo *ci = code_hdr->functions[i];
	if (erts_is_function_native(ci))
	    continue;
	clear_function_break(ci, ERTS_BPF_ALL);
    }

    erts_commit_staged_bp();

    for (i = 0; i < n; ++i) {
	ErtsCodeInfo *ci = code_hdr->functions[i];
	if (erts_is_function_native(ci))
	    continue;
	uninstall_breakpoint(ci);
	consolidate_bp_data(modp, ci, 1);
	ASSERT(ci->u.gen_bp == NULL);
    }
    return n;
}

static void *
signal_dispatcher_thread_func(void *unused)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_set_thread_name("signal_dispatcher");
#endif
    while (1) {
        union {int signum; char buf[4];} sb;
        Eterm signal;
	int res, i = 0;
	/* Block on read() waiting for a signal notification to arrive... */

        do {
            res = read(sig_notify_fds[0], (void *) &sb.buf[i], sizeof(int) - i);
            i += res > 0 ? res : 0;
        } while ((i < sizeof(int) && res >= 0) || (res < 0 && errno == EINTR));

	if (res < 0) {
	    erts_exit(ERTS_ABORT_EXIT,
		     "signal-dispatcher thread got unexpected error: %s (%d)\n",
		     erl_errno_id(errno),
		     errno);
	}
        /*
         * NOTE 1: The signal dispatcher thread should not do work
         *         that takes a substantial amount of time (except
         *         perhaps in test and debug builds). It needs to
         *         be responsive, i.e, it should only dispatch work
         *         to other threads.
         *
         * NOTE 2: The signal dispatcher thread is not a blockable
         *         thread (i.e., not a thread managed by the
         *         erl_thr_progress module). This is intentional.
         *         We want to be able to interrupt writing of a crash
         *         dump by hitting C-c twice. Since it isn't a
         *         blockable thread it is important that it doesn't
         *         change the state of any data that a blocking thread
         *         expects to have exclusive access to (unless the
         *         signal dispatcher itself explicitly is blocking all
         *         blockable threads).
         */
        switch (sb.signum) {
            case 0: continue;
            case SIGINT:
                break_requested();
                break;
            default:
                if ((signal = signum_to_signalterm(sb.signum)) == am_error) {
                    erts_exit(ERTS_ABORT_EXIT,
                            "signal-dispatcher thread received unknown "
                            "signal notification: '%d'\n",
                            sb.signum);
                }
                signal_notify_requested(signal);
        }
        ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
    }
    return NULL;
}

char *getenv_string(GETENV_STATE *state)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
    if (state->next_string[0] == '\0')
	return NULL;
    else {
	char *res = state->next_string;
	state->next_string += sys_strlen(res) + 1;
	return res;
    }
}

void
erts_consolidate_bif_bp_data(void)
{
    int i;

    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
    for (i = 0; i < BIF_SIZE; i++) {
	Export *ep = bif_export[i];
	consolidate_bp_data(0, ep->code+3, 0);
    }
}

ErtsMigrateResult
erts_port_migrate(Port *prt, int *prt_locked,
		  ErtsRunQueue *from_rq, int *from_locked,
		  ErtsRunQueue *to_rq, int *to_locked)
{
    ERTS_SMP_LC_ASSERT(*from_locked);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);

    ASSERT(!erts_common_run_queue);

    if (!*from_locked || !*to_locked) {
	if (from_rq < to_rq) {
	    if (!*to_locked) {
		if (!*from_locked)
		    erts_smp_runq_lock(from_rq);
		erts_smp_runq_lock(to_rq);
	    }
	    else if (erts_smp_runq_trylock(from_rq) == EBUSY) {
		erts_smp_runq_unlock(to_rq);
		erts_smp_runq_lock(from_rq);
		erts_smp_runq_lock(to_rq);
	    }
	}
	else {
	    if (!*from_locked) {
		if (!*to_locked)
		    erts_smp_runq_lock(to_rq);
		erts_smp_runq_lock(from_rq);
	    }
	    else if (erts_smp_runq_trylock(to_rq) == EBUSY) {
		erts_smp_runq_unlock(from_rq);
		erts_smp_runq_lock(to_rq);
		erts_smp_runq_lock(from_rq);
	    }
	}
	*to_locked = *from_locked = 1;
    }
    ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);

    /* Refuse to migrate to a suspended run queue */
    if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
	return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
    if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue))
	return ERTS_MIGRATE_FAILED_RUNQ_CHANGED;
    if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt))
	return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
    dequeue_port(from_rq, prt);
    erts_smp_atomic_set(&prt->run_queue, (long) to_rq);
    enqueue_port(to_rq, prt);
    erts_smp_notify_inc_runq(to_rq);
    return ERTS_MIGRATE_SUCCESS;
}

void
erts_clear_export_break(Module* modp, BeamInstr* pc)
{
    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());

    clear_function_break(pc, ERTS_BPF_ALL);
    erts_commit_staged_bp();
    *pc = (BeamInstr) 0;
    consolidate_bp_data(modp, pc, 0);
    ASSERT(pc[-4] == 0);
}

void
erts_clear_export_break(Module* modp, ErtsCodeInfo *ci)
{
    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());

    clear_function_break(ci, ERTS_BPF_ALL);
    erts_commit_staged_bp();
    *erts_codeinfo_to_code(ci) = (BeamInstr) 0;
    consolidate_bp_data(modp, ci, 0);
    ASSERT(ci->u.gen_bp == NULL);
}

Module*
erts_put_module(Eterm mod)
{
    Module e;
    int index;

    ASSERT(is_atom(mod));
    ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0));
    e.module = atom_val(mod);
    index = index_put(&module_table, (void*) &e);
    return (Module*) erts_index_lookup(&module_table, index);
}

void
erts_set_dist_entry_not_connected(DistEntry *dep)
{
    ERTS_SMP_LC_ASSERT(erts_lc_is_dist_entry_locked(dep));
    erts_smp_mtx_lock(&erts_dist_table_mtx);

    ASSERT(dep != erts_this_dist_entry);
    ASSERT(is_internal_port(dep->cid));

    if(dep->flags & DFLAG_PUBLISHED) {
	if(dep->prev) {
	    ASSERT(is_in_de_list(dep, erts_visible_dist_entries));
	    dep->prev->next = dep->next;
	}
	else {
	    ASSERT(erts_visible_dist_entries == dep);
	    erts_visible_dist_entries = dep->next;
	}

	ASSERT(erts_no_of_visible_dist_entries > 0);
	erts_no_of_visible_dist_entries--;
    }
    else {
	if(dep->prev) {
	    ASSERT(is_in_de_list(dep, erts_hidden_dist_entries));
	    dep->prev->next = dep->next;
	}
	else {
	    ASSERT(erts_hidden_dist_entries == dep);
	    erts_hidden_dist_entries = dep->next;
	}

	ASSERT(erts_no_of_hidden_dist_entries > 0);
	erts_no_of_hidden_dist_entries--;
    }

    if(dep->next)
	dep->next->prev = dep->prev;

    dep->status &= ~ERTS_DE_SFLG_CONNECTED;
    dep->flags = 0;
    dep->prev = NULL;
    dep->cid = NIL;

    dep->next = erts_not_connected_dist_entries;
    if(erts_not_connected_dist_entries) {
	ASSERT(erts_not_connected_dist_entries->prev == NULL);
	erts_not_connected_dist_entries->prev = dep;
    }
    erts_not_connected_dist_entries = dep;
    erts_no_of_not_connected_dist_entries++;
    erts_smp_mtx_unlock(&erts_dist_table_mtx);
}

void
erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
{
    ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);

    ASSERT(dep != erts_this_dist_entry);
    ASSERT(is_nil(dep->cid));
    ASSERT(is_internal_port(cid));

    if(dep->prev) {
	ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries));
	dep->prev->next = dep->next;
    }
    else {
	ASSERT(erts_not_connected_dist_entries == dep);
	erts_not_connected_dist_entries = dep->next;
    }

    if(dep->next)
	dep->next->prev = dep->prev;

    ASSERT(erts_no_of_not_connected_dist_entries > 0);
    erts_no_of_not_connected_dist_entries--;

    dep->status |= ERTS_DE_SFLG_CONNECTED;
    dep->flags = flags;
    dep->cid = cid;
    dep->connection_id++;
    dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK;
    dep->prev = NULL;

    if(flags & DFLAG_PUBLISHED) {
	dep->next = erts_visible_dist_entries;
	if(erts_visible_dist_entries) {
	    ASSERT(erts_visible_dist_entries->prev == NULL);
	    erts_visible_dist_entries->prev = dep;
	}
	erts_visible_dist_entries = dep;
	erts_no_of_visible_dist_entries++;
    }
    else {
	dep->next = erts_hidden_dist_entries;
	if(erts_hidden_dist_entries) {
	    ASSERT(erts_hidden_dist_entries->prev == NULL);
	    erts_hidden_dist_entries->prev = dep;
	}
	erts_hidden_dist_entries = dep;
	erts_no_of_hidden_dist_entries++;
    }
    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
}

void
erts_consolidate_bp_data(BpFunctions* f, int local)
{
    BpFunction* fs = f->matching;
    Uint i;
    Uint n = f->matched;

    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());

    for (i = 0; i < n; i++) {
	consolidate_bp_data(fs[i].mod, fs[i].pc, local);
    }
}

BIF_RETTYPE make_ref_0(BIF_ALIST_0)
{
    BIF_RETTYPE res;
    Eterm* hp;

    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));

    hp = HAlloc(BIF_P, REF_THING_SIZE);

    res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp);

    BIF_RET(res);
}

Eterm erts_make_ref(Process *c_p)
{
    Eterm* hp;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];

    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));

    hp = HAlloc(c_p, REF_THING_SIZE);

    make_ref_in_array(ref);
    write_ref_thing(hp, ref[0], ref[1], ref[2]);

    return make_internal_ref(hp);
}

void
erts_clear_mtrace_bif(Uint *pc) {
    BpDataTrace *bdt;
    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
    
    bdt = (BpDataTrace *) pc[-4];
    if (bdt) {
	if (bdt->match_spec) {
	    MatchSetUnref(bdt->match_spec);
	}
	Free(bdt);
    }
    pc[-4] = (Uint) NULL;
}

char *getenv_string(GETENV_STATE *state0)
{
   char **state = (char **) *state0;
   char *cp;

   ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));

   if (state == NULL)
      state = environ;

   cp = *state++;
   *state0 = (GETENV_STATE) state;

   return cp;
}