int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p, int n_rules)
{
    int i;

    if (!alg_p) {
        OPAL_OUTPUT((ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n"));
        return (-1);
    }

    OPAL_OUTPUT((ompi_coll_tuned_stream,"Number of algorithm rules %3d\n", n_rules));

    for (i=0;i<n_rules;i++) {
        ompi_coll_tuned_dump_alg_rule (&(alg_p[i]));
    }

    return (0);
}

int ompi_coll_tuned_allgatherv_intra_do_forced(const void *sbuf, int scount,
                                               struct ompi_datatype_t *sdtype,
                                               void *rbuf, const int *rcounts,
                                               const int *rdispls,
                                               struct ompi_datatype_t *rdtype,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
{
    mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;

    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "coll:tuned:allgatherv_intra_do_forced selected algorithm %d",
                 tuned_module->user_forced[ALLGATHERV].algorithm));

    switch (tuned_module->user_forced[ALLGATHERV].algorithm) {
    case (0):
        return ompi_coll_tuned_allgatherv_intra_dec_fixed(sbuf, scount, sdtype,
                                                          rbuf, rcounts, rdispls, rdtype,
                                                          comm, module);
    case (1):
        return ompi_coll_base_allgatherv_intra_basic_default(sbuf, scount, sdtype,
                                                             rbuf, rcounts, rdispls, rdtype,
                                                             comm, module);
    case (2):
        return ompi_coll_base_allgatherv_intra_bruck(sbuf, scount, sdtype,
                                                     rbuf, rcounts, rdispls, rdtype,
                                                     comm, module);
    case (3):
        return ompi_coll_base_allgatherv_intra_ring(sbuf, scount, sdtype,
                                                    rbuf, rcounts, rdispls, rdtype,
                                                    comm, module);
    case (4):
        return ompi_coll_base_allgatherv_intra_neighborexchange(sbuf, scount, sdtype,
                                                                rbuf, rcounts, rdispls, rdtype,
                                                                comm, module);
    case (5):
        return ompi_coll_base_allgatherv_intra_two_procs(sbuf, scount, sdtype,
                                                         rbuf, rcounts, rdispls, rdtype,
                                                         comm, module);
    } /* switch */
    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "coll:tuned:allgatherv_intra_do_forced attempt to select algorithm %d when only 0-%d is valid?",
                 tuned_module->user_forced[ALLGATHERV].algorithm,
                 ompi_coll_tuned_forced_max_algorithms[ALLGATHERV]));
    return (MPI_ERR_ARG);
}

int ompi_coll_tuned_get_target_method_params (ompi_coll_com_rule_t* base_com_rule, int mpi_msgsize, int *result_topo_faninout, 
                                              int* result_segsize)
{
    ompi_coll_msg_rule_t*  msg_p = (ompi_coll_msg_rule_t*) NULL;
    ompi_coll_msg_rule_t*  best_msg_p = (ompi_coll_msg_rule_t*) NULL;
    int i, best;

    if (!base_com_rule) {
        return (0);
    }

    if (!result_topo_faninout) {
        return (0);
    }

    if (!result_segsize) {
        return (0);
    }

    if (!base_com_rule->n_msg_sizes) {   /* check for count of message sizes */
        return (0);    /* no msg sizes so no rule */
    }

    /* ok have some msg sizes, now to find the one closest to my mpi_msgsize */
   
    /* make a copy of the first msg rule */
    best_msg_p = msg_p = base_com_rule->msg_rules;
    i = best = 0;

    while (i<base_com_rule->n_msg_sizes) {
        /*       OPAL_OUTPUT((ompi_coll_tuned_stream,"checking mpi_msgsize %d against com_id %d msg_id %d index %d msg_size %d",  */
        /*             mpi_msgsize, msg_p->com_rule_id, msg_p->msg_rule_id, i, msg_p->msg_size)); */
        if (msg_p->msg_size <= mpi_msgsize) {
            best = i;
            best_msg_p = msg_p;
            /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":ok\n")); */
        }
        else {
            /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":nop\n")); */
            break;
        }
        /* go to the next entry */
        msg_p++;
        i++;
    }

    OPAL_OUTPUT((ompi_coll_tuned_stream,"Selected the following msg rule id %d\n", best_msg_p->msg_rule_id));
    ompi_coll_tuned_dump_msg_rule (best_msg_p);

    /* return the segment size */
    *result_topo_faninout = best_msg_p->result_topo_faninout;

    /* return the segment size */
    *result_segsize = best_msg_p->result_segsize;

    /* return the algorithm/method to use */
    return (best_msg_p->result_alg);  
}

static int
cleanup_scatter_handles(ompi_coll_portals4_request_t *request)
{
    int ret, line;

    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                 "coll:portals4:cleanup_scatter_handles enter rank %d", request->u.scatter.my_rank));

    /**********************************/
    /* Cleanup Scatter Handles             */
    /**********************************/
    do {
        ret = PtlMEUnlink(request->u.scatter.scatter_meh);
        if (PTL_IN_USE == ret) {
            opal_output(ompi_coll_base_framework.framework_output,
                        "%s:%4d: scatter_meh still in use (ret=%d, rank %2d)",
                        __FILE__, __LINE__, ret, request->u.scatter.my_rank);
            continue;
        }
        if (PTL_OK != ret) {
            ret = OMPI_ERROR;
            line = __LINE__;
            goto err_hdlr;
        }
    } while (ret == PTL_IN_USE);

    ret = PtlCTFree(request->u.scatter.scatter_cth);
    if (PTL_OK != ret) {
        ret = OMPI_ERROR;
        line = __LINE__;
        goto err_hdlr;
    }

    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                 "coll:portals4:cleanup_scatter_handles exit rank %d", request->u.scatter.my_rank));

    return OMPI_SUCCESS;

err_hdlr:
    opal_output(ompi_coll_base_framework.framework_output,
                "%s:%4d:%4d\tError occurred ret=%d, rank %2d",
                __FILE__, __LINE__, line, ret, request->u.scatter.my_rank);

    return ret;
}

/* i.e. alg table and dynamic changable rules if allocated etc */
static int tuned_close(void)
{
    sdn_finalize();

    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: called"));

    /* dealloc alg table if allocated */
    /* dealloc dynamic changable rules if allocated */

    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: done!"));

    if( NULL != mca_coll_tuned_component.all_base_rules ) {
        ompi_coll_tuned_free_all_rules(mca_coll_tuned_component.all_base_rules, COLLCOUNT);
        mca_coll_tuned_component.all_base_rules = NULL;
    }

    return OMPI_SUCCESS;
}

static void main_thread_event_callback(int fd, short event, void *context)
{
    cmd_t cmd;

    OPAL_OUTPUT((-1, "main thread -- reading command"));
    opal_fd_read(pipe_to_main_thread[0], cmd_size, &cmd);
    switch (cmd.pc_cmd) {
    case CMD_CALL_FUNCTION:
        OPAL_OUTPUT((-1, "fd main thread: calling command"));
        main_pipe_cmd_call_function(&cmd);
        break;

    default:
        OPAL_OUTPUT((-1, "fd main thread: unknown pipe command: %d",
                    cmd.pc_cmd));
        break;
    }
}

/**
 * This graph API tell us if two vertices are adjacent
 *
 * @param graph The graph that the vertices belongs to.
 * @param vertex1 first vertex.
 * @param vertex2 second vertex.
 *
 * @return uint32_t the weight of the connection between the two
 *         vertices or infinity if the vertices are not
 *         connected.
 */
uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2)
{
    opal_adjacency_list_t *adj_list;
    opal_list_item_t *item;
    opal_graph_edge_t *edge;

    /**
     * Verify that the first vertex belongs to the graph.
     */
    if (graph != vertex1->in_graph) {
        OPAL_OUTPUT((0,"opal_graph_adjacent 1 Vertex1 %p not in the graph %p\n",(void *)vertex1,(void *)graph));
        return DISTANCE_INFINITY;
    }
    /**
     * Verify that the second vertex belongs to the graph.
     */
    if (graph != vertex2->in_graph) {
        OPAL_OUTPUT((0,"opal_graph_adjacent 2 Vertex2 %p not in the graph %p\n",(void *)vertex2,(void *)graph));
        return DISTANCE_INFINITY;
    }
    /**
     * If the first vertex and the second vertex are the same
     * vertex, the distance between the is 0.
     */
    if (vertex1 == vertex2) {
        return 0;
    }
    /**
     * find the second vertex in the adjacency list of the first
     * vertex.
     */
    adj_list = (opal_adjacency_list_t *) vertex1->in_adj_list;
    for (item = opal_list_get_first(adj_list->edges);
         item != opal_list_get_end(adj_list->edges);
         item  = opal_list_get_next(item)) {
        edge = (opal_graph_edge_t *)item;
        if (edge->end == vertex2) {
            /* if the second vertex was found in the adjacency list of the first one, return the weight */
            return edge->weight;
        }
    }
    /* if the second vertex was not found in the adjacency list of the first one, return infinity */
    return DISTANCE_INFINITY;
}

int ompi_coll_base_barrier_intra_basic_linear(struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module)
{
    int i, err, rank, size, line;
    ompi_request_t** requests = NULL;

    rank = ompi_comm_rank(comm);
    size = ompi_comm_size(comm);

    /* All non-root send & receive zero-length message. */
    if (rank > 0) {
        err = MCA_PML_CALL(send (NULL, 0, MPI_BYTE, 0,
                                 MCA_COLL_BASE_TAG_BARRIER,
                                 MCA_PML_BASE_SEND_STANDARD, comm));
        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }

        err = MCA_PML_CALL(recv (NULL, 0, MPI_BYTE, 0,
                                 MCA_COLL_BASE_TAG_BARRIER,
                                 comm, MPI_STATUS_IGNORE));
        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
    }

    /* The root collects and broadcasts the messages. */

    else {
        requests = coll_base_comm_get_reqs(module->base_data, size);
        if( NULL == requests ) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto err_hndl; }

        for (i = 1; i < size; ++i) {
            err = MCA_PML_CALL(irecv(NULL, 0, MPI_BYTE, MPI_ANY_SOURCE,
                                     MCA_COLL_BASE_TAG_BARRIER, comm,
                                     &(requests[i])));
            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
        }
        err = ompi_request_wait_all( size-1, requests+1, MPI_STATUSES_IGNORE );
        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
        requests = NULL;  /* we're done the requests array is clean */

        for (i = 1; i < size; ++i) {
            err = MCA_PML_CALL(send(NULL, 0, MPI_BYTE, i,
                                    MCA_COLL_BASE_TAG_BARRIER,
                                    MCA_PML_BASE_SEND_STANDARD, comm));
            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
        }
    }

    /* All done */
    return MPI_SUCCESS;
 err_hndl:
    OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
                  __FILE__, line, err, rank) );
    (void)line;  // silence compiler warning
    if( NULL != requests )
        ompi_coll_base_free_reqs(requests, size);
    return err;
}

int ompi_coll_tuned_reduce_intra_do_forced(void *sbuf, void* rbuf, int count,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op, int root,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module)
{
    mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
    mca_coll_tuned_comm_t *data = tuned_module->tuned_data;

    const int segsize      = data->user_forced[REDUCE].segsize;
    const int chain_fanout = data->user_forced[REDUCE].chain_fanout;
    const int max_requests = data->user_forced[REDUCE].max_requests;

    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_intra_do_forced selected algorithm %d", 
                 data->user_forced[REDUCE].algorithm));


    switch (data->user_forced[REDUCE].algorithm) {
    case (0):  return ompi_coll_tuned_reduce_intra_dec_fixed (sbuf, rbuf, count, dtype, 
                                                              op, root, comm, module);
    case (1):  return ompi_coll_tuned_reduce_intra_basic_linear (sbuf, rbuf, count, dtype,
                                                                 op, root, comm, module);
    case (2):  return ompi_coll_tuned_reduce_intra_chain (sbuf, rbuf, count, dtype,
                                                          op, root, comm, module,
                                                          segsize, chain_fanout, max_requests);
    case (3):  return ompi_coll_tuned_reduce_intra_pipeline (sbuf, rbuf, count, dtype,
                                                             op, root, comm, module,
                                                             segsize, max_requests);
    case (4):  return ompi_coll_tuned_reduce_intra_binary (sbuf, rbuf, count, dtype,
                                                           op, root, comm, module,
                                                           segsize, max_requests);
    case (5):  return ompi_coll_tuned_reduce_intra_binomial (sbuf, rbuf, count, dtype,
                                                             op, root, comm, module,
                                                             segsize, max_requests);
    case (6):  return ompi_coll_tuned_reduce_intra_in_order_binary(sbuf, rbuf, count, dtype,
                                                                   op, root, comm, module,
                                                                   segsize, max_requests);
    default:
        OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_intra_do_forced attempt to select algorithm %d when only 0-%d is valid?",
                     data->user_forced[REDUCE].algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCE]));
        return (MPI_ERR_ARG);
    } /* switch */
}

static void cts_sent(mca_btl_base_module_t* btl,
                     struct mca_btl_base_endpoint_t* ep,
                     struct mca_btl_base_descriptor_t* des,
                     int status)
{
    /* Nothing to do/empty function (we can't pass in a NULL pointer
       for the des_cbfunc) */
    OPAL_OUTPUT((-1, "CTS send to %s completed",
                 opal_get_proc_hostname(ep->endpoint_proc->proc_opal)));
}

uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2)
{
    opal_value_array_t *distance_array;
    uint32_t items_in_distance_array, spf = DISTANCE_INFINITY;
    vertex_distance_from_t *vertex_distance;
    uint32_t i;

    /**
     * Verify that the first vertex belongs to the graph.
     */
    if (graph != vertex1->in_graph) {
        OPAL_OUTPUT((0,"opal_graph_spf 1 Vertex1 %p not in the graph %p\n",(void *)vertex1,(void *)graph));
        return DISTANCE_INFINITY;
    }
    /**
     * Verify that the second vertex belongs to the graph.
     */
    if (graph != vertex2->in_graph) {
        OPAL_OUTPUT((0,"opal_graph_spf 2 Vertex2 %p not in the graph %p\n",(void *)vertex2,(void *)graph));
        return DISTANCE_INFINITY;
    }
    /**
     * Run Dijkstra algorithm on the graph from the start vertex.
     */
    distance_array = OBJ_NEW(opal_value_array_t);
    opal_value_array_init(distance_array, sizeof(vertex_distance_from_t));
    opal_value_array_reserve(distance_array,50);
    items_in_distance_array = opal_graph_dijkstra(graph, vertex1, distance_array);
    /**
     * find the end vertex in the distance array that Dijkstra
     * algorithm returned.
     */
    for (i = 0; i < items_in_distance_array; i++) {
        vertex_distance = opal_value_array_get_item(distance_array, i);
        if (vertex_distance->vertex == vertex2) {
            spf = vertex_distance->weight;
            break;
        }
    }
    OBJ_RELEASE(distance_array);
    /* return the distance (weight) to the end vertex */
    return spf;
}

int
opal_progress_set_event_flag(int flag)
{
    int tmp = opal_progress_event_flag;
    opal_progress_event_flag = flag;

    OPAL_OUTPUT((debug_output, "progress: set_event_flag setting to %d", flag));

    return tmp;
}

int ompi_coll_tuned_allgatherv_intra_dec_dynamic(const void *sbuf, int scount,
                                                 struct ompi_datatype_t *sdtype,
                                                 void* rbuf, const int *rcounts,
                                                 const int *rdispls,
                                                 struct ompi_datatype_t *rdtype,
                                                 struct ompi_communicator_t *comm,
                                                 mca_coll_base_module_t *module)
{
    mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;

    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "ompi_coll_tuned_allgatherv_intra_dec_dynamic"));

    if (tuned_module->com_rules[ALLGATHERV]) {
        /* We have file based rules:
           - calculate message size and other necessary information */
        int comsize, i;
        int alg, faninout, segsize, ignoreme;
        size_t dsize, total_size;

        comsize = ompi_comm_size(comm);
        ompi_datatype_type_size (sdtype, &dsize);
        total_size = 0;
        for (i = 0; i < comsize; i++) { total_size += dsize * rcounts[i]; }

        alg = ompi_coll_tuned_get_target_method_params (tuned_module->com_rules[ALLGATHERV],
                                                        total_size, &faninout, &segsize, &ignoreme);
        if (alg) {
            /* we have found a valid choice from the file based rules for
               this message size */
            return ompi_coll_tuned_allgatherv_intra_do_this (sbuf, scount, sdtype,
                                                             rbuf, rcounts,
                                                             rdispls, rdtype,
                                                             comm, module,
                                                             alg, faninout, segsize);
        }
    }

    /* We do not have file based rules */
    if (tuned_module->user_forced[ALLGATHERV].algorithm) {
        /* User-forced algorithm */
        return ompi_coll_tuned_allgatherv_intra_do_this(sbuf, scount, sdtype,
                                                        rbuf, rcounts, rdispls, rdtype,
                                                        comm, module,
                                                        tuned_module->user_forced[ALLGATHERV].algorithm,
                                                        tuned_module->user_forced[ALLGATHERV].tree_fanout,
                                                        tuned_module->user_forced[ALLGATHERV].segsize);
    }

    /* Use default decision */
    return ompi_coll_tuned_allgatherv_intra_dec_fixed (sbuf, scount, sdtype,
                                                       rbuf, rcounts,
                                                       rdispls, rdtype,
                                                       comm, module);
}

/*
 * Another recursive doubling type algorithm, but in this case
 * we go up the tree and back down the tree.  
 */
int ompi_coll_tuned_barrier_intra_tree(struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
{
    int rank, size, depth, err, jump, partner;

    rank = ompi_comm_rank(comm);
    size = ompi_comm_size(comm);
    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "ompi_coll_tuned_barrier_intra_tree %d", 
                 rank));

    /* Find the nearest power of 2 of the communicator size. */
    depth = opal_next_poweroftwo_inclusive(size);

    for (jump=1; jump<depth; jump<<=1) {
        partner = rank ^ jump;
        if (!(partner & (jump-1)) && partner < size) {
            if (partner > rank) {
                err = MCA_PML_CALL(recv (NULL, 0, MPI_BYTE, partner, 
                                         MCA_COLL_BASE_TAG_BARRIER, comm,
                                         MPI_STATUS_IGNORE));
                if (MPI_SUCCESS != err)
                    return err;
            } else if (partner < rank) {
                err = MCA_PML_CALL(send (NULL, 0, MPI_BYTE, partner,
                                         MCA_COLL_BASE_TAG_BARRIER, 
                                         MCA_PML_BASE_SEND_STANDARD, comm));
                if (MPI_SUCCESS != err)
                    return err;
            }
        }
    }
    
    depth >>= 1;
    for (jump = depth; jump>0; jump>>=1) {
        partner = rank ^ jump;
        if (!(partner & (jump-1)) && partner < size) {
            if (partner > rank) {
                err = MCA_PML_CALL(send (NULL, 0, MPI_BYTE, partner,
                                         MCA_COLL_BASE_TAG_BARRIER,
                                         MCA_PML_BASE_SEND_STANDARD, comm));
                if (MPI_SUCCESS != err)
                    return err;
            } else if (partner < rank) {
                err = MCA_PML_CALL(recv (NULL, 0, MPI_BYTE, partner, 
                                         MCA_COLL_BASE_TAG_BARRIER, comm,
                                         MPI_STATUS_IGNORE));
                if (MPI_SUCCESS != err)
                    return err;
            }
        }
    }

    return MPI_SUCCESS;
}

/*
 * Invoked when there's a new communicator that has been created.
 * Look at the communicator and decide which set of functions and
 * priority we want to return.
 */
const mca_coll_base_module_1_0_0_t *
ompi_coll_tuned_comm_query(struct ompi_communicator_t *comm, int *priority,
                           struct mca_coll_base_comm_t **data)
{
    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:module_tuned query called"));

    *priority = ompi_coll_tuned_priority;

    /* 
     * Choose whether to use [intra|inter] decision functions 
     * and if using fixed OR dynamic rule sets.
     * Right now you cannot mix them, maybe later on it can be changed
     * but this would probably add an extra if and funct call to the path
     */

    if (OMPI_COMM_IS_INTER(comm)) {
        if (ompi_coll_tuned_use_dynamic_rules) {
            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_query using inter_dynamic"));
            to_use = &inter_dynamic;
        } else {
            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_query using inter_fixed"));
            to_use = &inter_fixed;
        }
    } else { /* is an intra comm */
        /**
         * If the communicator size is less than 2 we have specialized modules
         * to handle the intra collective communications.
         */
        if( ompi_comm_size(comm) < 2) {
            *priority = 0;
            return NULL;
        }
        if (ompi_coll_tuned_use_dynamic_rules) {
            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_query using intra_dynamic"));
            to_use = &intra_dynamic;
        } else {
            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_query using intra_fixed"));
            to_use = &intra_fixed;
        }
    }
    return to_use;
}

int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module,
                                           int algorithm, int faninout, int segsize)
{
    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:barrier_intra_do_this selected algorithm %d topo fanin/out%d", algorithm, faninout));

    switch (algorithm) {
    case (0):   return ompi_coll_tuned_barrier_intra_dec_fixed (comm, module);
    case (1):   return ompi_coll_tuned_barrier_intra_basic_linear (comm, module); 
    case (2):   return ompi_coll_tuned_barrier_intra_doublering (comm, module);
    case (3):   return ompi_coll_tuned_barrier_intra_recursivedoubling (comm, module);
    case (4):   return ompi_coll_tuned_barrier_intra_bruck (comm, module);
    case (5):   return ompi_coll_tuned_barrier_intra_two_procs (comm, module);
    case (6):   return ompi_coll_tuned_barrier_intra_tree (comm, module);
    default:
        OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
                     algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
        return (MPI_ERR_ARG);
    } /* switch */
}

bool
opal_progress_set_yield_when_idle(bool yieldopt)
{
    bool tmp = opal_progress_yield_when_idle;
    opal_progress_yield_when_idle = (yieldopt) ? 1 : 0;

    OPAL_OUTPUT((debug_output, "progress: progress_set_yield_when_idle to %s",
                                    opal_progress_yield_when_idle ? "true" : "false"));

    return tmp;
}

bool
opal_progress_set_yield_when_idle(bool yieldopt)
{
    bool tmp = (call_yield == 0) ? false : true;
    call_yield = (yieldopt) ? 1 : 0;

    OPAL_OUTPUT((debug_output, "progress: progress_set_yield_when_idle to %s",
                                    call_yield == 0 ? "false" : "true"));

    return tmp;
}

int
ompi_coll_tuned_gather_intra_do_forced(void *sbuf, int scount,
                                       struct ompi_datatype_t *sdtype,
                                       void* rbuf, int rcount,
                                       struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
{
    mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;

    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "coll:tuned:gather_intra_do_forced selected algorithm %d",
                 tuned_module->user_forced[GATHER].algorithm));

    switch (tuned_module->user_forced[GATHER].algorithm) {
    case (0):
        return ompi_coll_tuned_gather_intra_dec_fixed(sbuf, scount, sdtype,
                                                      rbuf, rcount, rdtype,
                                                      root, comm, module);
    case (1):
        return ompi_coll_base_gather_intra_basic_linear(sbuf, scount, sdtype,
                                                        rbuf, rcount, rdtype,
                                                        root, comm, module);
    case (2):
        return ompi_coll_base_gather_intra_binomial(sbuf, scount, sdtype,
                                                     rbuf, rcount, rdtype,
                                                     root, comm, module);
    case (3):
        return ompi_coll_base_gather_intra_linear_sync(sbuf, scount, sdtype,
                                                       rbuf, rcount, rdtype,
                                                       root, comm, module,
                                                       tuned_module->user_forced[GATHER].segsize);
    } /* switch */
    OPAL_OUTPUT((ompi_coll_tuned_stream,
                 "coll:tuned:gather_intra_do_forced attempt to select algorithm %d when only 0-%d is valid?",
                 tuned_module->user_forced[GATHER].algorithm,
                 ompi_coll_tuned_forced_max_algorithms[GATHER]));
    return (MPI_ERR_ARG);
}

/*
 * Send CTS control fragment
 */
void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint)
{
    mca_btl_openib_send_control_frag_t *sc_frag;
    mca_btl_base_descriptor_t *base_des;
    mca_btl_openib_frag_t *openib_frag;
    mca_btl_openib_com_frag_t *com_frag;
    mca_btl_openib_control_header_t *ctl_hdr;

    OPAL_OUTPUT((-1, "SENDING CTS to %s on qp index %d (QP num %d)",
                 (NULL == endpoint->endpoint_proc->proc_ompi->proc_hostname) ?
                 "unknown" : endpoint->endpoint_proc->proc_ompi->proc_hostname,
                 mca_btl_openib_component.credits_qp,
                 endpoint->qps[mca_btl_openib_component.credits_qp].qp->lcl_qp->qp_num));
    sc_frag = alloc_control_frag(endpoint->endpoint_btl);
    if (OPAL_UNLIKELY(NULL == sc_frag)) {
        BTL_ERROR(("Failed to allocate control buffer"));
        mca_btl_openib_endpoint_invoke_error(endpoint);
        return;
    }

    /* I dislike using the "to_<foo>()" macros; I prefer using the
       explicit member fields to ensure I get the types right.  Since
       this is not a performance-criticial part of the code, it's
       ok. */
    com_frag = &(sc_frag->super.super);
    openib_frag = &(com_frag->super);
    base_des = &(openib_frag->base);

    base_des->des_cbfunc = cts_sent;
    base_des->des_cbdata = NULL;
    base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
    base_des->order = mca_btl_openib_component.credits_qp;
    openib_frag->segment.base.seg_len = sizeof(mca_btl_openib_control_header_t);
    com_frag->endpoint = endpoint;

    sc_frag->hdr->tag = MCA_BTL_TAG_IB;
    sc_frag->hdr->cm_seen = 0;
    sc_frag->hdr->credits = 0;

    ctl_hdr = (mca_btl_openib_control_header_t*)
        openib_frag->segment.base.seg_addr.pval;
    ctl_hdr->type = MCA_BTL_OPENIB_CONTROL_CTS;

    /* Send the fragment */
    OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
    if (OMPI_SUCCESS != mca_btl_openib_endpoint_post_send(endpoint, sc_frag)) {
        BTL_ERROR(("Failed to post CTS send"));
        mca_btl_openib_endpoint_invoke_error(endpoint);
    }
    endpoint->endpoint_cts_sent = true;
    OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
}