int
mca_io_romio314_file_iwrite (ompi_file_t *fh,
                          const void *buf,
                          int count,
                          struct ompi_datatype_t *datatype,
                          ompi_request_t **request)
{
    int ret;
    mca_io_romio314_data_t *data;

    data = (mca_io_romio314_data_t *) fh->f_io_selected_data;
    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
    ret =
        ROMIO_PREFIX(MPI_File_iwrite) (data->romio_fh, buf, count, datatype,
                                       request);
    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);

    return ret;
}

int
mca_io_romio314_file_write_all (ompi_file_t *fh,
                             const void *buf,
                             int count,
                             struct ompi_datatype_t *datatype,
                             ompi_status_public_t * status)
{
    int         ret;
    mca_io_romio314_data_t *data;

    data = (mca_io_romio314_data_t *) fh->f_io_selected_data;
    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
    ret =
        ROMIO_PREFIX(MPI_File_write_all) (data->romio_fh, buf, count, datatype,
                                         status);
    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);

    return ret;
}

pid_t
orte_waitpid(pid_t wpid, int *status, int options)
{
    opal_process_handle_t* pending;

    OPAL_THREAD_LOCK(&mutex);

    /**
     * Is the child already gone ?
     */
    pending = find_pending_pid( wpid, false );
    if( NULL != pending ) {
        *status = pending->status;
        opal_list_remove_item( &pending_pids, (opal_list_item_t*)pending );
        OBJ_RELEASE(pending);
        OPAL_THREAD_UNLOCK(&mutex);
        return wpid;
    }

    /**
     * Do we have any registered callback for this particular pid ?
     */
    pending = find_pending_cb( wpid, false );
    if( NULL != pending ) {
        opal_list_remove_item( &registered_cb, (opal_list_item_t*)pending );
        OBJ_RELEASE( pending );
    }

    /**
     * No luck so far. Wait until the process complete ...
     */
    if( WAIT_OBJECT_0 == WaitForSingleObject( (HANDLE)wpid, INFINITE ) ) {
        DWORD exitCode;
        /* Process completed. Grab the exit value and return. */
        if( 0 == GetExitCodeProcess( (HANDLE)wpid, &exitCode ) ) {
            int error = GetLastError();
        }
        *status = (int)exitCode;
    }
    OPAL_THREAD_UNLOCK(&mutex);
    return wpid;
}

/*
 *  Initiate the appropriate action based on the state of the connection
 *  to the peer.
 *
 */
int mca_oob_tcp_peer_send(mca_oob_tcp_peer_t* peer, mca_oob_tcp_msg_t* msg)
{
    int rc = ORTE_SUCCESS;
    OPAL_THREAD_LOCK(&peer->peer_lock);
    switch(peer->peer_state) {
    case MCA_OOB_TCP_CONNECTING:
    case MCA_OOB_TCP_CONNECT_ACK:
    case MCA_OOB_TCP_CLOSED:
    case MCA_OOB_TCP_RESOLVE:
        /*
         * queue the message and attempt to resolve the peer address
         */
        opal_list_append(&peer->peer_send_queue, (opal_list_item_t*)msg);
        if(peer->peer_state == MCA_OOB_TCP_CLOSED) {
            peer->peer_state = MCA_OOB_TCP_RESOLVE;
            OPAL_THREAD_UNLOCK(&peer->peer_lock);
            return mca_oob_tcp_resolve(peer);
        }
        break;
    case MCA_OOB_TCP_FAILED:
        rc = ORTE_ERR_UNREACH;
        break;
    case MCA_OOB_TCP_CONNECTED:
        /*
         * start the message and queue if not completed 
         */
        if (NULL != peer->peer_send_msg) {
            opal_list_append(&peer->peer_send_queue, (opal_list_item_t*)msg);
        } else {
            /*if the send does not complete */
            if(!mca_oob_tcp_msg_send_handler(msg, peer)) {
                peer->peer_send_msg = msg;
                opal_event_add(&peer->peer_send_event, 0);
            } else {
                mca_oob_tcp_msg_complete(msg, &peer->peer_name);
            }
        }
        break;
    }
    OPAL_THREAD_UNLOCK(&peer->peer_lock);
    return rc;
}

int oshmem_proc_pack(oshmem_proc_t **proclist,
                     int proclistsize,
                     opal_buffer_t* buf)
{
    int i, rc;

    OPAL_THREAD_LOCK(&oshmem_proc_lock);

    /* cycle through the provided array, packing the OSHMEM level
     * data for each proc. This data may or may not be included
     * in any subsequent modex operation, so we include it here
     * to ensure completion of a connect/accept handshake. See
     * the ompi/mca/dpm framework for an example of where and how
     * this info is used.
     *
     * Eventually, we will review the procedures that call this
     * function to see if duplication of communication can be
     * reduced. For now, just go ahead and pack the info so it
     * can be sent.
     */
    for (i = 0; i < proclistsize; i++) {
        rc = opal_dss.pack(buf, &(proclist[i]->super.proc_name), 1, ORTE_NAME);
        if (rc != ORTE_SUCCESS) {
            ORTE_ERROR_LOG(rc);
            OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
            return rc;
        }
        rc = opal_dss.pack(buf, &(proclist[i]->super.proc_arch), 1, OPAL_UINT32);
        if (rc != ORTE_SUCCESS) {
            ORTE_ERROR_LOG(rc);
            OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
            return rc;
        }
        rc = opal_dss.pack(buf, &(proclist[i]->super.proc_hostname), 1, OPAL_STRING);
        if (rc != ORTE_SUCCESS) {
            ORTE_ERROR_LOG(rc);
            OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
            return rc;
        }
    } OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
    return OSHMEM_SUCCESS;
}

void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* tcp_proc)
{
    if( NULL != tcp_proc->proc_opal ) {
        /* remove from list of all proc instances */
        OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
        opal_proc_table_remove_value(&mca_btl_tcp_component.tcp_procs,
                                     tcp_proc->proc_opal->proc_name);
        OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
        OBJ_RELEASE(tcp_proc->proc_opal);
        tcp_proc->proc_opal = NULL;
    }
    /* release resources */
    if(NULL != tcp_proc->proc_endpoints) {
        free(tcp_proc->proc_endpoints);
    }
    if(NULL != tcp_proc->proc_addrs) {
        free(tcp_proc->proc_addrs);
    }
    OBJ_DESTRUCT(&tcp_proc->proc_lock);
}

int orte_pls_base_orted_cancel_operation(void)
{
    /* protect for threads */
    OPAL_THREAD_LOCK(&orte_pls_base.orted_cmd_lock);
    
    /* cancel any waiting receive - we don't want to hear it */
    orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_PLS_ORTED_ACK);
    
    /* set the completion status to reflect cancellation -- no need to
       print anything */
    completion_status = ORTE_ERR_SILENT;
    
    /* declare us "done" so we can exit cleanly */
    opal_condition_signal(&orte_pls_base.orted_cmd_cond);
    
    /* unlock us */
    OPAL_THREAD_UNLOCK(&orte_pls_base.orted_cmd_lock);
    
    return ORTE_SUCCESS;
}

static int finalize(void)
{
    int rc;
    opal_list_item_t *item;
    
    OPAL_THREAD_LOCK(&mca_iof_orted_component.lock);
    while ((item = opal_list_remove_first(&mca_iof_orted_component.sinks)) != NULL) {
        OBJ_RELEASE(item);
    }
    OBJ_DESTRUCT(&mca_iof_orted_component.sinks);
    while ((item = opal_list_remove_first(&mca_iof_orted_component.procs)) != NULL) {
        OBJ_RELEASE(item);
    }
    OBJ_DESTRUCT(&mca_iof_orted_component.procs);
    /* Cancel the RML receive */
    rc = orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_IOF_PROXY);
    OPAL_THREAD_UNLOCK(&mca_iof_orted_component.lock);
    OBJ_DESTRUCT(&mca_iof_orted_component.lock);
    return rc;
}

/**
 * This function resize the free_list to contain at least the specified
 * number of elements. We do not create all of them in the same memory
 * segment. Instead we will several time the fl_num_per_alloc elements
 * until we reach the required number of the maximum allowed by the 
 * initialization.
 */
int
ompi_free_list_resize(ompi_free_list_t* flist, size_t size)
{
    ssize_t inc_num;
    int ret = OMPI_SUCCESS;

    if (flist->fl_num_allocated > size) {
        return OMPI_SUCCESS;
    }
    OPAL_THREAD_LOCK(&((flist)->fl_lock));
    inc_num = size - flist->fl_num_allocated;
    while( inc_num > 0 ) {
        ret = ompi_free_list_grow(flist, flist->fl_num_per_alloc);
        if( OMPI_SUCCESS != ret ) break;
        inc_num = size - flist->fl_num_allocated;
    }
    OPAL_THREAD_UNLOCK(&((flist)->fl_lock));

    return ret;
}

int
mca_io_romio_file_iread_at (ompi_file_t *fh,
                            MPI_Offset offset,
                            void *buf,
                            int count,
                            struct ompi_datatype_t *datatype,
                            ompi_request_t **request)
{
    int ret;
    mca_io_romio_data_t *data;

    data = (mca_io_romio_data_t *) fh->f_io_selected_data;
    OPAL_THREAD_LOCK (&mca_io_romio_mutex);
    ret =
        ROMIO_PREFIX(MPI_File_iread_at) (data->romio_fh, offset, buf, count,
                                         datatype, request);
    OPAL_THREAD_UNLOCK (&mca_io_romio_mutex);

    return ret;
}

int
mca_io_romio314_file_get_view (ompi_file_t *fh,
                            MPI_Offset * disp,
                            struct ompi_datatype_t ** etype,
                            struct ompi_datatype_t ** filetype,
                            char *datarep)
{
    int ret;
    mca_io_romio314_data_t *data;

    data = (mca_io_romio314_data_t *) fh->f_io_selected_data;
    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
    ret =
        ROMIO_PREFIX(MPI_File_get_view) (data->romio_fh, disp, etype, filetype,
                                        datarep);
    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);

    return ret;

}

ompi_proc_t * ompi_proc_find ( const ompi_process_name_t * name )
{
    ompi_proc_t *proc, *rproc=NULL;
    ompi_rte_cmp_bitmask_t mask;

    /* return the proc-struct which matches this jobid+process id */
    mask = OMPI_RTE_CMP_JOBID | OMPI_RTE_CMP_VPID;
    OPAL_THREAD_LOCK(&ompi_proc_lock);
    for(proc =  (ompi_proc_t*)opal_list_get_first(&ompi_proc_list);
        proc != (ompi_proc_t*)opal_list_get_end(&ompi_proc_list);
        proc =  (ompi_proc_t*)opal_list_get_next(proc)) {
        if (OPAL_EQUAL == ompi_rte_compare_name_fields(mask, &proc->proc_name, name)) {
            rproc = proc;
            break;
        }
    }
    OPAL_THREAD_UNLOCK(&ompi_proc_lock);

    return rproc;
}

static inline int32_t
create_send_tag(ompi_osc_pt2pt_module_t *module)
{
#if OMPI_HAVE_THREAD_SUPPORT && OPAL_HAVE_ATOMIC_CMPSET_32
    int32_t newval, oldval;
    do {
        oldval = module->p2p_tag_counter;
        newval = (oldval + 1) % mca_pml.pml_max_tag;
    } while (0 == opal_atomic_cmpset_32(&module->p2p_tag_counter, oldval, newval));
    return newval;
#else
    int32_t ret;
    /* no compare and swap - have to lock the module */
    OPAL_THREAD_LOCK(&module->p2p_lock);
    module->p2p_tag_counter = (module->p2p_tag_counter + 1) % mca_pml.pml_max_tag;
    ret = module->p2p_tag_counter;
    OPAL_THREAD_UNLOCK(&module->p2p_lock);
    return ret;
#endif
}

bool mca_oob_ud_req_is_in_list (mca_oob_ud_req_t *req, opal_list_t *list)
{
    opal_list_item_t *item;
    bool rc = false;

    OPAL_THREAD_LOCK(&mca_oob_ud_component.ud_match_lock);

    for (item = opal_list_get_first (list) ;
         item != opal_list_get_end (list) ;
         item = opal_list_get_next (item)) {
        if (item == (opal_list_item_t *) req) {
            rc = true;
            break;
        }
    }

    OPAL_THREAD_UNLOCK(&mca_oob_ud_component.ud_match_lock);

    return rc;
}

int
orte_wait_cb_cancel(pid_t wpid)
{
    opal_process_handle_t* pending;

    OPAL_THREAD_LOCK(&mutex);

    /**
     * Do we have any registered callback for this particular pid ?
     */
    pending = find_pending_cb( wpid, false );
    if( NULL != pending ) {
        opal_list_remove_item( &registered_cb, (opal_list_item_t*)pending );
        OBJ_RELEASE( pending );
        OPAL_THREAD_UNLOCK(&mutex);
        return ORTE_SUCCESS;
    }
    OPAL_THREAD_UNLOCK(&mutex);
    return ORTE_ERR_BAD_PARAM;
}

int orte_iof_base_callback_delete(
    const orte_process_name_t* proc,
    int tag)
{
    orte_iof_base_endpoint_t* endpoint;
    opal_list_item_t* item;

    OPAL_THREAD_LOCK(&orte_iof_base.iof_lock);
    if(NULL == (endpoint = orte_iof_base_endpoint_lookup(proc,ORTE_IOF_SINK, tag))) {
        OPAL_THREAD_UNLOCK(&orte_iof_base.iof_lock);
        return ORTE_ERR_NOT_FOUND;
    }

    while(NULL != (item = opal_list_remove_first(&endpoint->ep_callbacks))) {
        OBJ_RELEASE(item);
    }
    OBJ_RELEASE(endpoint);
    OPAL_THREAD_UNLOCK(&orte_iof_base.iof_lock);
    return ORTE_SUCCESS;
}

/**********************************************************************
 *
 * Receive an accumulate on the target side
 *
 **********************************************************************/
static void
ompi_osc_pt2pt_sendreq_recv_accum_long_cb(ompi_osc_pt2pt_longreq_t *longreq)
{
    ompi_osc_pt2pt_send_header_t *header = 
        (ompi_osc_pt2pt_send_header_t*) longreq->req_comp_cbdata;
    void *payload = (void*) (header + 1);
    int ret;

    /* lock the window for accumulates */
    OPAL_THREAD_LOCK(&longreq->req_module->p2p_acc_lock);

    opal_list_remove_item(&(longreq->req_module->p2p_long_msgs), 
                          &(longreq->super.super));

    /* copy the data from the temporary buffer into the user window */
    ret = ompi_osc_pt2pt_process_op(longreq->req_module, 
                                    header, 
                                    longreq->req_datatype, 
                                    longreq->req_op, 
                                    payload,
                                    header->hdr_msg_length);

    /* unlock the window for accumulates */
    OPAL_THREAD_UNLOCK(&longreq->req_module->p2p_acc_lock);
    
    opal_output_verbose(50, ompi_osc_base_output,
                        "%d finished receiving long accum message from %d",
                        longreq->req_module->p2p_comm->c_my_rank, 
                        header->hdr_origin);               

    /* free the temp buffer */
    free(longreq->req_comp_cbdata);

    /* Release datatype & op */
    OBJ_RELEASE(longreq->req_datatype);
    OBJ_RELEASE(longreq->req_op);

    OPAL_THREAD_ADD32(&(longreq->req_module->p2p_num_pending_in), -1);

    ompi_osc_pt2pt_longreq_free(longreq);
}

int mca_io_ompio_file_get_byte_offset (ompi_file_t *fh,
                                       OMPI_MPI_OFFSET_TYPE offset,
                                       OMPI_MPI_OFFSET_TYPE *disp)
{
    mca_io_ompio_data_t *data;
    int i, k, index;
    size_t temp_offset;

    data = (mca_io_ompio_data_t *) fh->f_io_selected_data;

    OPAL_THREAD_LOCK(&fh->f_lock);
    temp_offset = data->ompio_fh.f_view_extent *
        (offset*data->ompio_fh.f_etype_size / data->ompio_fh.f_view_size);
    

    i = (offset*data->ompio_fh.f_etype_size) % data->ompio_fh.f_view_size;
    index = 0;
    k = 0;

    while (1) {
        k = data->ompio_fh.f_decoded_iov[index].iov_len;
        if (i >= k) {
            i -= k;
            index++;
            if ( 0 == i ) {
                k=0;
                break;
            }
        }
        else {
            k=i;
            break;
        }
    }

    *disp = data->ompio_fh.f_disp + temp_offset +
        (OMPI_MPI_OFFSET_TYPE)(intptr_t)data->ompio_fh.f_decoded_iov[index].iov_base + k;
    OPAL_THREAD_UNLOCK(&fh->f_lock);

    return OMPI_SUCCESS;
}

/* in some cases, all PE procs are required to do a modex so they
 * can (at the least) exchange their architecture. Since we cannot
 * know in advance if this was required, we provide a separate function
 * to set the arch (instead of doing it inside of oshmem_proc_init) that
 * can be called after the modex completes in oshmem_shmem_init. Thus, we
 * know that - regardless of how the arch is known, whether via modex
 * or dropped in from a local daemon - the arch can be set correctly
 * at this time
 */
int oshmem_proc_set_arch(void)
{
    oshmem_proc_t *proc = NULL;
    opal_list_item_t *item = NULL;
    int ret = OSHMEM_SUCCESS;

    OPAL_THREAD_LOCK(&oshmem_proc_lock);

    for (item = opal_list_get_first(&oshmem_proc_list);
            item != opal_list_get_end(&oshmem_proc_list);
            item = opal_list_get_next(item)) {
        proc = (oshmem_proc_t*) item;

        if (OSHMEM_PROC_VPID(proc) != ORTE_PROC_MY_NAME->vpid) {
            /* if arch is different than mine, create a new convertor for this proc */
            if (proc->super.proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
                OBJ_RELEASE(proc->super.proc_convertor);
                proc->super.proc_convertor = opal_convertor_create(proc->super.proc_arch, 0);
#else
                orte_show_help("help-shmem-runtime.txt",
                               "heterogeneous-support-unavailable",
                               true,
                               orte_process_info.nodename,
                               proc->super.proc_hostname == NULL ?
                                       "<hostname unavailable>" :
                                       proc->super.proc_hostname);
                OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
                return OSHMEM_ERR_NOT_SUPPORTED;
#endif
            }
        }
    }

    /* Set predefined groups */
    ret = oshmem_proc_group_init();

    OPAL_THREAD_UNLOCK(&oshmem_proc_lock);

    return ret;
}

int orte_grpcomm_base_set_proc_attr(const char *attr_name,
                                    const void *data,
                                    size_t size)
{
    int rc;
    
    OPAL_THREAD_LOCK(&mutex);
    
    OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base_output,
                         "%s grpcomm:set_proc_attr: setting attribute %s data size %lu",
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                         attr_name, (unsigned long)size));

    /* Pack the attribute name information into the local buffer */
    if (ORTE_SUCCESS != (rc = opal_dss.pack(modex_buffer, &attr_name, 1, OPAL_STRING))) {
        ORTE_ERROR_LOG(rc);
        goto cleanup;
    }

    /* pack the size */
    if (ORTE_SUCCESS != (rc = opal_dss.pack(modex_buffer, &size, 1, OPAL_SIZE))) {
        ORTE_ERROR_LOG(rc);
        goto cleanup;
    }
    
    /* Pack the actual data into the buffer */
    if (0 != size) {
        if (ORTE_SUCCESS != (rc = opal_dss.pack(modex_buffer, (void *) data, size, OPAL_BYTE))) {
            ORTE_ERROR_LOG(rc);
            goto cleanup;
        }
    }
    
    /* track the number of entries */
    ++num_entries;
    
cleanup:
    OPAL_THREAD_UNLOCK(&mutex);
    
    return rc;
}