orte_proc_t* orte_rmaps_base_setup_proc(orte_job_t *jdata,
                                        orte_node_t *node,
                                        orte_app_idx_t idx)
{
    orte_proc_t *proc;
    int rc;

    proc = OBJ_NEW(orte_proc_t);
    /* set the jobid */
    proc->name.jobid = jdata->jobid;
    /* flag the proc as ready for launch */
    proc->state = ORTE_PROC_STATE_INIT;
    proc->app_idx = idx;

    OBJ_RETAIN(node);  /* maintain accounting on object */    
    proc->node = node;
    proc->nodename = node->name;
    node->num_procs++;
    if (node->slots_inuse < node->slots) {
        node->slots_inuse += orte_rmaps_base.cpus_per_rank;
    }
    if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
        ORTE_ERROR_LOG(rc);
        OBJ_RELEASE(proc);
        return NULL;
    }
    /* retain the proc struct so that we correctly track its release */
    OBJ_RETAIN(proc);

    return proc;
}

int orte_rmaps_base_add_proc_to_map(orte_job_map_t *map, orte_node_t *node,
                                    bool oversubscribed, orte_proc_t *proc)
{
    orte_std_cntr_t i;
    orte_node_t *node_from_map;
    int rc;
    
    /* see if this node has already been assigned to the map - if
     * not, then add the pointer to the pointer array
     */
    for (i=0; i < map->nodes->size; i++) {
        if (NULL == (node_from_map = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
            continue;
        }
        if (node_from_map->index == node->index) {
            /* we have this node in the array */
            goto PROCESS;
        }
    }
    /* if we get here, then this node isn't already in the map - add it */
    OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base.rmaps_output,
                         "%s rmaps:base: adding node %s to map",
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                         (NULL == node->name) ? "NULL" : node->name));
    
    if (ORTE_SUCCESS > (rc = opal_pointer_array_add(map->nodes, (void*)node))) {
        ORTE_ERROR_LOG(rc);
        return rc;
    }
    OBJ_RETAIN(node);  /* maintain accounting on object */
    ++map->num_nodes;
    
PROCESS:
    /* add the proc to this node's local processes - it is assumed
     * that the proc isn't already there as this would be an error
     * in the mapper
     */
    OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base.rmaps_output,
                         "%s rmaps:base: mapping proc for job %s to node %s whose daemon is %s",
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                         ORTE_JOBID_PRINT(proc->name.jobid),
                         (NULL == node->name) ? "NULL" : node->name,
                         (NULL == node->daemon) ? "NULL" : ORTE_NAME_PRINT(&(node->daemon->name))));
    
    if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
        ORTE_ERROR_LOG(rc);
        return rc;
    }
    /* retain the proc struct so that we correctly track its release */
    OBJ_RETAIN(proc);
    ++node->num_procs;

    /* update the oversubscribed state of the node */
    node->oversubscribed = oversubscribed;
    
    return ORTE_SUCCESS;
}

/*
 * Back end to MPI_FILE_OPEN
 */
int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
                   int amode, struct ompi_info_t *info, ompi_file_t **fh)
{
    int ret;
    ompi_file_t *file;

    file = OBJ_NEW(ompi_file_t);
    if (NULL == file) {
        return OMPI_ERR_OUT_OF_RESOURCE;
    }


    /* Save the params */

    file->f_comm = comm;
    OBJ_RETAIN(comm);

    if (MPI_INFO_NULL != info) {
        if(NULL == file->f_info) {
            file->f_info = OBJ_NEW(ompi_info_t);
        }
        if (OMPI_SUCCESS != (ret = ompi_info_dup(info, &file->f_info))) {
            OBJ_RELEASE(file);
            return ret;
        }
    } else {
        file->f_info = MPI_INFO_NULL;
        OBJ_RETAIN(MPI_INFO_NULL);
    }

    file->f_amode = amode;
    file->f_filename = strdup(filename);
    if (NULL == file->f_filename) {
        OBJ_RELEASE(file);
        return OMPI_ERR_OUT_OF_RESOURCE;
    }

    /* Create the mutex */
    OBJ_CONSTRUCT(&file->f_mutex, opal_mutex_t);

    /* Select a module and actually open the file */

    if (OMPI_SUCCESS != (ret = mca_io_base_file_select(file, NULL))) {
        OBJ_RELEASE(file);
        return ret;
    }

    /* All done */

    *fh = file;
    return OMPI_SUCCESS;
}

static int alloc_window(struct ompi_communicator_t *comm, ompi_info_t *info, int flavor, ompi_win_t **win_out)
{
    ompi_win_t *win;
    ompi_group_t *group;
    int acc_ops, flag, ret;

    /* create the object */
    win = OBJ_NEW(ompi_win_t);
    if (NULL == win) {
        return OMPI_ERR_OUT_OF_RESOURCE;
    }

    ret = ompi_info_get_value_enum (info, "accumulate_ops", &acc_ops,
                                    OMPI_WIN_ACCUMULATE_OPS_SAME_OP_NO_OP,
                                    ompi_win_accumulate_ops, &flag);
    if (OMPI_SUCCESS != ret) {
        OBJ_RELEASE(win);
        return ret;
    }

    win->w_acc_ops = acc_ops;
    win->w_flavor = flavor;

    /* setup data that is independent of osc component */
    group = comm->c_local_group;
    OBJ_RETAIN(group);
    win->w_group = group;

    *win_out = win;

    return OMPI_SUCCESS;
}

int ompi_grequest_start(
    MPI_Grequest_query_function *gquery_fn,
    MPI_Grequest_free_function *gfree_fn,
    MPI_Grequest_cancel_function *gcancel_fn,
    void* gstate,
    ompi_request_t** request)
{
    ompi_grequest_t *greq = OBJ_NEW(ompi_grequest_t);
    if(greq == NULL) {
        return OMPI_ERR_OUT_OF_RESOURCE;
    }
    /* We call RETAIN here specifically to increase the refcount to 2.
       See comment before the destructor for an explanation. */
    OBJ_RETAIN(greq);

    greq->greq_base.req_state = OMPI_REQUEST_ACTIVE;
    greq->greq_state = gstate;
    greq->greq_query.c_query = gquery_fn;
    greq->greq_free.c_free = gfree_fn;
    greq->greq_cancel.c_cancel = gcancel_fn; 
    greq->greq_base.req_status = ompi_status_empty;

    *request = &greq->greq_base;
    return OMPI_SUCCESS;
}

int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandler)
{
    MPI_Errhandler tmp;

    OPAL_CR_NOOP_PROGRESS();

    if (MPI_PARAM_CHECK) {
        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
        if (ompi_win_invalid(win)) {
            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN,
                                          FUNC_NAME);
        } else if (NULL == errhandler) {
            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG,
                                          FUNC_NAME);
        }
    }

    /* On 64 bits environments we have to make sure the reading of the
       error_handler became atomic. */
    do {
        tmp = win->error_handler;
    } while (!OPAL_ATOMIC_CMPSET_PTR(&(win->error_handler), tmp, tmp));

    /* Retain the errhandler, corresponding to object refcount
       decrease in errhandler_free.c. */
    OBJ_RETAIN(win->error_handler);
    *errhandler = win->error_handler;

    /* All done */
    return MPI_SUCCESS;
}

int
ompi_osc_pt2pt_module_post(ompi_group_t *group,
                           int assert,
                           ompi_win_t *win)
{
    int i;
    ompi_osc_pt2pt_module_t *module = P2P_MODULE(win);

    OBJ_RETAIN(group);
    ompi_group_increment_proc_count(group);

    OPAL_THREAD_LOCK(&(module->p2p_lock));
    assert(NULL == module->p2p_pw_group);
    module->p2p_pw_group = group;    

    /* Set our mode to expose w/ post */
    ompi_win_remove_mode(win, OMPI_WIN_FENCE);
    ompi_win_append_mode(win, OMPI_WIN_EXPOSE_EPOCH | OMPI_WIN_POSTED);

    /* list how many complete counters we're still waiting on */
    module->p2p_num_complete_msgs +=
        ompi_group_size(module->p2p_pw_group);
    OPAL_THREAD_UNLOCK(&(module->p2p_lock));

    /* send a hello counter to everyone in group */
    for (i = 0 ; i < ompi_group_size(module->p2p_pw_group) ; ++i) {
        ompi_osc_pt2pt_control_send(module, 
                                    ompi_group_peer_lookup(group, i),
                                    OMPI_OSC_PT2PT_HDR_POST, 1, 0);
    }

    return OMPI_SUCCESS;
}

int orte_iof_base_callback_create(
    const orte_process_name_t* proc,
    int tag,
    orte_iof_base_callback_fn_t cbfunc,
    void *cbdata)
{
    orte_iof_base_callback_t* cb = OBJ_NEW(orte_iof_base_callback_t);
    orte_iof_base_endpoint_t* endpoint;
    if(NULL == cb)
        return ORTE_ERR_OUT_OF_RESOURCE;

    OPAL_THREAD_LOCK(&orte_iof_base.iof_lock);
    if((endpoint = orte_iof_base_endpoint_lookup(proc,ORTE_IOF_SINK,tag)) == NULL) {
        endpoint = OBJ_NEW(orte_iof_base_endpoint_t);
        if(NULL == endpoint) {
            OPAL_THREAD_UNLOCK(&orte_iof_base.iof_lock);
            return ORTE_ERR_OUT_OF_RESOURCE;
        }
        endpoint->ep_origin = *proc;
        endpoint->ep_mode = ORTE_IOF_SINK;
        endpoint->ep_tag = tag;
        endpoint->ep_fd = -1;
        opal_list_append(&orte_iof_base.iof_endpoints, &endpoint->super);
    } else {
        OBJ_RETAIN(endpoint);
    }
    cb->cb_func = cbfunc;
    cb->cb_data = cbdata;
    opal_list_append(&endpoint->ep_callbacks, (opal_list_item_t*)cb);
    OPAL_THREAD_UNLOCK(&orte_iof_base.iof_lock);
    return ORTE_SUCCESS;
}

int
ompi_win_group(ompi_win_t *win, ompi_group_t **group) {
    OBJ_RETAIN(win->w_group);
    *group = win->w_group;

    return OMPI_SUCCESS;
}

int
ompi_win_init(void)
{
    int ret;

    assert (sizeof (ompi_predefined_win_t) >= sizeof (ompi_win_t));

    /* setup window Fortran array */
    OBJ_CONSTRUCT(&ompi_mpi_windows, opal_pointer_array_t);
    if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_mpi_windows, 4,
                                                OMPI_FORTRAN_HANDLE_MAX, 16) ) {
        return OMPI_ERROR;
    }

    /* Setup MPI_WIN_NULL */
    OBJ_CONSTRUCT(&ompi_mpi_win_null.win, ompi_win_t);
    ompi_mpi_win_null.win.w_flags = OMPI_WIN_INVALID;
    ompi_mpi_win_null.win.w_group = &ompi_mpi_group_null.group;
    OBJ_RETAIN(&ompi_mpi_group_null);
    ompi_win_set_name(&ompi_mpi_win_null.win, "MPI_WIN_NULL");
    opal_pointer_array_set_item(&ompi_mpi_windows, 0, &ompi_mpi_win_null.win);

    ret = mca_base_var_enum_create ("accumulate_ops", accumulate_ops_values, &ompi_win_accumulate_ops);
    if (OPAL_SUCCESS != ret) {
        return ret;
    }

    ret = mca_base_var_enum_create_flag ("accumulate_order", accumulate_order_flags, &ompi_win_accumulate_order);
    if (OPAL_SUCCESS != ret) {
        return ret;
    }

    return OMPI_SUCCESS;
}

static int link_items(repository_item_t *src, repository_item_t *depend)
{
  dependency_item_t *di;

  /* Bozo check */

  if (NULL == src || NULL == depend) {
    return OPAL_ERR_BAD_PARAM;
  }

  /* Make a new depedency item */

  di = OBJ_NEW(dependency_item_t);
  if (NULL == di) {
    return OPAL_ERR_OUT_OF_RESOURCE;
  }

  /* Initialize the new dependency item */

  di->di_repository_entry = depend;

  /* Add it to the dependency list on the source repository entry */

  opal_list_append(&src->ri_dependencies, (opal_list_item_t *) di);

  /* Increment the refcount in the dependency */

  OBJ_RETAIN(depend);

  /* All done */

  return OPAL_SUCCESS;
}

int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler) 
{
    MPI_Errhandler tmp;

    OPAL_CR_NOOP_PROGRESS();

    if (MPI_PARAM_CHECK) {
        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);

        if (ompi_win_invalid(win)) {
            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN,
                                          FUNC_NAME);
        } else if (NULL == errhandler ||
                   MPI_ERRHANDLER_NULL == errhandler ||
                   (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type && 
                    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
        }
    }

    /* Prepare the new error handler */
    OBJ_RETAIN(errhandler);

    /* Ditch the old errhandler, and decrement its refcount.  On 64
       bits environments we have to make sure the reading of the
       error_handler became atomic. */
    do {
        tmp = win->error_handler;
    } while (!OPAL_ATOMIC_CMPSET(&(win->error_handler), tmp, errhandler));
    OBJ_RELEASE(tmp);

    /* All done */
    return MPI_SUCCESS;
}

int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler)
{
    MPI_Errhandler tmp;

    OPAL_CR_NOOP_PROGRESS();

    if (MPI_PARAM_CHECK) {
        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);

        if (ompi_win_invalid(win)) {
            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_WIN,
                                          FUNC_NAME);
        } else if (NULL == errhandler ||
                   MPI_ERRHANDLER_NULL == errhandler ||
                   (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type &&
                    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
        }
    }

    /* Prepare the new error handler */
    OBJ_RETAIN(errhandler);

    OPAL_THREAD_LOCK(&win->w_lock);
    /* Ditch the old errhandler, and decrement its refcount. */
    tmp = win->error_handler;
    win->error_handler = errhandler;
    OBJ_RELEASE(tmp);
    OPAL_THREAD_UNLOCK(&win->w_lock);

    /* All done */
    return MPI_SUCCESS;
}

/**
 * JOB
 */
int orte_dt_copy_job(orte_job_t **dest, orte_job_t *src, opal_data_type_t type)
{
    (*dest) = src;
    OBJ_RETAIN(src);
    
    return ORTE_SUCCESS;
}

/*
 *  Initializes the mpool module.
 */
void mca_mpool_grdma_module_init(mca_mpool_grdma_module_t* mpool, mca_mpool_grdma_pool_t *pool)
{
    OBJ_RETAIN(pool);
    mpool->pool = pool;

    mpool->super.mpool_component = &mca_mpool_grdma_component.super;
    mpool->super.mpool_base = NULL; /* no base .. */
    mpool->super.mpool_alloc = mca_mpool_grdma_alloc;
    mpool->super.mpool_realloc = mca_mpool_grdma_realloc;
    mpool->super.mpool_free = mca_mpool_grdma_free;
    mpool->super.mpool_register = mca_mpool_grdma_register;
    mpool->super.mpool_find = mca_mpool_grdma_find;
    mpool->super.mpool_deregister = mca_mpool_grdma_deregister;
    mpool->super.mpool_release_memory = mca_mpool_grdma_release_memory;
    mpool->super.mpool_finalize = mca_mpool_grdma_finalize;
    mpool->super.mpool_ft_event = mca_mpool_grdma_ft_event;
    mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
    mpool->super.rcache = pool->rcache;

    mpool->stat_cache_hit = mpool->stat_cache_miss = mpool->stat_evicted = 0;
    mpool->stat_cache_found = mpool->stat_cache_notfound = 0;

    OBJ_CONSTRUCT(&mpool->reg_list, ompi_free_list_t);
    ompi_free_list_init_new(&mpool->reg_list, mpool->resources.sizeof_reg,
                            opal_cache_line_size,
                            OBJ_CLASS(mca_mpool_base_registration_t), 
                            0, opal_cache_line_size, 0, -1, 32, NULL);
}

int MPI_File_get_errhandler( MPI_File file, MPI_Errhandler *errhandler)
{
    /* Error checking */

    if (MPI_PARAM_CHECK) {
        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);

        /* Note that MPI-2:9.7 (p265 in the ps; 261 in the pdf) explicitly
           says that you are allowed to set the error handler on
           MPI_FILE_NULL */

        if (NULL == file) {
            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_FILE,
                                          "MPI_File_get_errhandler");
        } else if (NULL == errhandler) {
            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                          "MPI_File_get_errhandler");
        }
    }

    /* Retain the errhandler, corresponding to object refcount
       decrease in errhandler_free.c. */
    OBJ_RETAIN(file->error_handler);
    *errhandler = file->error_handler;

    /* All done */

    return MPI_SUCCESS;
}

int
mca_io_romio314_file_close (ompi_file_t *fh)
{
    int ret;
    mca_io_romio314_data_t *data;

    /* If we've already started MPI_Finalize by this point, then just
       give up (because ROMIO's file close routine calls MPI_Barrier,
       which we obviously can't do if we've started to MPI_Finalize).
       The user didn't close the file, so they should expect
       unexpected behavior. */
    if (ompi_mpi_finalized) {
        return OMPI_SUCCESS;
    }

    /* Because ROMIO expects the MPI library to provide error handler
     * management routines but it doesn't ever participate in
     * MPI_File_close, we have to somehow inform the MPI library that
     * we no longer hold a reference to any user defined error
     * handler.  We do this by setting the errhandler at this point to
     * MPI_ERRORS_RETURN. */
    if (fh->error_handler != &ompi_mpi_errors_return.eh) {
        OBJ_RELEASE(fh->error_handler);
        fh->error_handler = &ompi_mpi_errors_return.eh;
        OBJ_RETAIN(fh->error_handler);
    }

    data = (mca_io_romio314_data_t *) fh->f_io_selected_data;

    OPAL_THREAD_LOCK (&mca_io_romio314_mutex);
    ret = ROMIO_PREFIX(MPI_File_close) (&data->romio_fh);
    OPAL_THREAD_UNLOCK (&mca_io_romio314_mutex);

    return ret;
}

int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler) 
{
  /* Error checking */

  if (MPI_PARAM_CHECK) {
    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
    if (ompi_comm_invalid(comm)) {
      return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM,
                                    FUNC_NAME);
    } else if (NULL == errhandler ||
               MPI_ERRHANDLER_NULL == errhandler ||
               ( OMPI_ERRHANDLER_TYPE_COMM != errhandler->eh_mpi_object_type &&
		 OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
      return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
                                    FUNC_NAME);
    }
  }

  /* Ditch the old errhandler, and decrement its refcount */

  OBJ_RELEASE(comm->error_handler);

  /* We have a valid comm and errhandler, so increment its refcount */

  comm->error_handler = errhandler;
  OBJ_RETAIN(comm->error_handler);

  /* All done */
  
  return MPI_SUCCESS;
}

int ompi_group_incl_bmap(ompi_group_t* group, int n, const int *ranks,
                         ompi_group_t **new_group)
{
    /* local variables */
    int my_group_rank,i,bit_set;
    ompi_group_t *group_pointer, *new_group_pointer;

    group_pointer = (ompi_group_t *)group;

    if ( 0 == n ) {
        *new_group = MPI_GROUP_EMPTY;
        OBJ_RETAIN(MPI_GROUP_EMPTY);
        return OMPI_SUCCESS;
    }

    new_group_pointer = ompi_group_allocate_bmap(group->grp_proc_count, n);
    if( NULL == new_group_pointer ) {
        return MPI_ERR_GROUP;
    }
    /* Initialize the bit array to zeros */
    for (i=0 ; i<new_group_pointer->sparse_data.grp_bitmap.grp_bitmap_array_len ; i++) {
        new_group_pointer->
            sparse_data.grp_bitmap.grp_bitmap_array[i] = 0;
    }

    /* set the bits */
    for (i=0 ; i<n ; i++) {
        bit_set = ranks[i] % BSIZE;
        new_group_pointer->
            sparse_data.grp_bitmap.grp_bitmap_array[(int)(ranks[i]/BSIZE)] |= (1 << bit_set);
    }

    new_group_pointer -> grp_parent_group_ptr = group_pointer;

    OBJ_RETAIN(new_group_pointer -> grp_parent_group_ptr);
    ompi_group_increment_proc_count(new_group_pointer -> grp_parent_group_ptr);

    ompi_group_increment_proc_count(new_group_pointer);
    my_group_rank=group_pointer->grp_my_rank;

    ompi_group_translate_ranks (group_pointer,1,&my_group_rank,
                                new_group_pointer,&new_group_pointer->grp_my_rank);

    *new_group = (MPI_Group)new_group_pointer;

    return OMPI_SUCCESS;
}

int
ompi_win_group(ompi_win_t *win, ompi_group_t **group) {
    OBJ_RETAIN(win->w_group);
    ompi_group_increment_proc_count(win->w_group);
    *group = win->w_group;

    return OMPI_SUCCESS;
}