static inline int mca_btl_ugni_ep_connect_start (mca_btl_base_endpoint_t *ep) {
    int rc;

    rc = mca_btl_ugni_ep_connect_rdma (ep);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        return rc;
    }

    BTL_VERBOSE(("initiaiting connection to remote peer with address: %u id: %u proc: %p",
                 ep->common->ep_rem_addr, ep->common->ep_rem_id, (void *)ep->peer_proc));

    /* bind endpoint to remote address */
    /* we bind two endpoints to seperate out local smsg completion and local fma completion */
    rc = opal_common_ugni_ep_create (ep->common, ep->btl->smsg_local_cq, &ep->smsg_ep_handle);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        return rc;
    }

    /* build connection data */
    rc = mca_btl_ugni_ep_smsg_get_mbox (ep);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        return rc;
    }

    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTING;

    memset (&ep->remote_attr, 0, sizeof (ep->remote_attr));

    BTL_VERBOSE(("btl/ugni connection to remote peer initiated"));

    return OPAL_SUCCESS;
}

mca_btl_base_descriptor_t *
mca_btl_scif_alloc(struct mca_btl_base_module_t *btl,
                   struct mca_btl_base_endpoint_t *endpoint,
                   uint8_t order, size_t size, uint32_t flags)
{
    mca_btl_scif_base_frag_t *frag = NULL;

    BTL_VERBOSE(("allocating fragment of size: %u", (unsigned int)size));

    if (size <= mca_btl_scif_module.super.btl_eager_limit) {
        (void) MCA_BTL_SCIF_FRAG_ALLOC_EAGER(endpoint, frag);
    }

    if (OPAL_UNLIKELY(NULL == frag)) {
        return NULL;
    }

    BTL_VERBOSE(("btl/scif_module allocated frag of size: %u, flags: %x. frag = %p",
                 (unsigned int)size, flags, (void *) frag));

    frag->base.des_flags = flags;
    frag->base.order = order;
    frag->base.des_segments = frag->segments;
    frag->base.des_segment_count = 1;

    frag->segments[0].seg_len       = size;

    return &frag->base;
}

static inline int mca_btl_scif_ep_get_buffer (mca_btl_base_endpoint_t *ep) {
    int rc;

    rc = posix_memalign ((void **) &ep->recv_buffer.buffer, opal_getpagesize(), mca_btl_scif_component.segment_size);
    if (0 > rc) {
        return OPAL_ERR_OUT_OF_RESOURCE;
    }

    memset (ep->recv_buffer.buffer, 0, mca_btl_scif_component.segment_size);

    ep->recv_buffer.scif_offset = scif_register (ep->scif_epd, ep->recv_buffer.buffer,
                                                 mca_btl_scif_component.segment_size, 0,
                                                 SCIF_PROT_READ | SCIF_PROT_WRITE, 0);
    if (SCIF_REGISTER_FAILED == ep->recv_buffer.scif_offset) {
        BTL_VERBOSE(("failed to register a scif buffer of size %d. errno = %d",
                     mca_btl_scif_component.segment_size, errno));
        free (ep->recv_buffer.buffer);
        ep->recv_buffer.buffer = NULL;
        return OPAL_ERROR;
    }

    ep->recv_buffer.startp = (uint32_t *) ep->recv_buffer.buffer;
    ep->recv_buffer.endp   = ep->recv_buffer.startp + 1;

    ep->recv_buffer.startp[0] = ep->recv_buffer.endp[0] = 64;

    BTL_VERBOSE(("allocated buffer of size %d bytes. with scif registration %lu",
                 mca_btl_scif_component.segment_size, (unsigned long) ep->recv_buffer.scif_offset));

    return OPAL_SUCCESS;
}

static void *mca_btl_scif_connect_accept (void *arg)
{
    struct scif_pollepd pollepd = {.epd = mca_btl_scif_module.scif_fd, .events = SCIF_POLLIN, .revents = 0};
    int rc;

    BTL_VERBOSE(("btl/scif: listening for new connections"));

    /* listen for connections */
    while (1) {
        pollepd.revents = 0;

        rc = scif_poll (&pollepd, 1, -1);
        if (1 == rc) {
            if (SCIF_POLLIN != pollepd.revents) {
                break;
            }

            rc = mca_btl_scif_ep_connect_start_passive ();
            if (OMPI_SUCCESS != rc) {
                BTL_VERBOSE(("btl/scif: error accepting scif connection"));
                continue;
            }
        } else {
            break;
        }
    }

    BTL_VERBOSE(("btl/scif: stopped listening for new connections"));

    return NULL;
}

int mca_btl_scif_del_procs (struct mca_btl_base_module_t *btl,
                            size_t nprocs, struct ompi_proc_t **procs,
                            struct mca_btl_base_endpoint_t **peers) {
    /* do nothing for now */
    return OMPI_SUCCESS;
}

static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
{
    mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
    size_t size = (size_t)((uintptr_t) reg->bound - (uintptr_t) reg->base);
    int i;

    /* register the fragment with all connected endpoints */
    for (i = 0 ; i < (int) mca_btl_scif_module.endpoint_count ; ++i) {
        if ((off_t)-1 != scif_reg->registrations[i] &&
            MCA_BTL_SCIF_EP_STATE_CONNECTED == mca_btl_scif_module.endpoints[i].state) {
            (void) scif_unregister(mca_btl_scif_module.endpoints[i].scif_epd,
                                   scif_reg->registrations[i], size);
        }
    }

    free (scif_reg->registrations);

    return OMPI_SUCCESS;
}

/* send the eager rdma connect message to the remote endpoint */
static int mca_btl_openib_endpoint_send_eager_rdma(
    mca_btl_base_endpoint_t* endpoint)
{
    mca_btl_openib_module_t* openib_btl = endpoint->endpoint_btl;
    mca_btl_openib_eager_rdma_header_t *rdma_hdr;
    mca_btl_openib_send_control_frag_t* frag;
    int rc;

    frag = alloc_control_frag(openib_btl);
    if(NULL == frag) {
        return -1;
    }

    to_base_frag(frag)->base.des_cbfunc =
        mca_btl_openib_endpoint_eager_rdma_connect_cb;
    to_base_frag(frag)->base.des_cbdata = NULL;
    to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
    to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
    to_base_frag(frag)->segment.seg_len =
        sizeof(mca_btl_openib_eager_rdma_header_t);
    to_com_frag(frag)->endpoint = endpoint;

    frag->hdr->tag = MCA_BTL_TAG_IB;
    rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
    rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA;
    rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey;
    rdma_hdr->rdma_start.lval = opal_ptr_ptol(endpoint->eager_rdma_local.base.pval);
    BTL_VERBOSE(("sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64
                 ", pval %p, ival %" PRIu32 " type %d and sizeof(rdma_hdr) %d\n",
                 rdma_hdr->rkey,
                 rdma_hdr->rdma_start.lval,
                 rdma_hdr->rdma_start.pval,
                 rdma_hdr->rdma_start.ival,
                 rdma_hdr->control.type,
                 (int) sizeof(mca_btl_openib_eager_rdma_header_t)
                 ));

    if(endpoint->nbo) {
        BTL_OPENIB_EAGER_RDMA_CONTROL_HEADER_HTON((*rdma_hdr));

        BTL_VERBOSE(("after HTON: sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32 "\n",
                     rdma_hdr->rkey,
                     rdma_hdr->rdma_start.lval,
                     rdma_hdr->rdma_start.pval,
                     rdma_hdr->rdma_start.ival
                     ));
    }
    rc = mca_btl_openib_endpoint_send(endpoint, frag);
    if (OPAL_SUCCESS == rc || OPAL_ERR_RESOURCE_BUSY == rc)
        return OPAL_SUCCESS;

    MCA_BTL_IB_FRAG_RETURN(frag);
    BTL_ERROR(("Error sending RDMA buffer: %s", strerror(errno)));
    return rc;
}

static inline int mca_btl_ugni_ep_connect_finish (mca_btl_base_endpoint_t *ep) {
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
    gni_return_t grc;
    int rc;

    BTL_VERBOSE(("finishing connection. remote attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
                 "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
                 "msg_maxsize = %d", ep->remote_attr->smsg_attr.msg_type, ep->remote_attr->smsg_attr.msg_buffer,
                 ep->remote_attr->smsg_attr.buff_size, ep->remote_attr->smsg_attr.mem_hndl.qword1,
                 ep->remote_attr->smsg_attr.mem_hndl.qword2, ep->remote_attr->smsg_attr.mbox_offset,
                 ep->remote_attr->smsg_attr.mbox_maxcredit, ep->remote_attr->smsg_attr.msg_maxsize));

    BTL_VERBOSE(("finishing connection. local attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
                 "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
                 "msg_maxsize = %d", ep->mailbox->attr.smsg_attr.msg_type, ep->mailbox->attr.smsg_attr.msg_buffer,
                 ep->mailbox->attr.smsg_attr.buff_size, ep->mailbox->attr.smsg_attr.mem_hndl.qword1,
                 ep->mailbox->attr.smsg_attr.mem_hndl.qword2, ep->mailbox->attr.smsg_attr.mbox_offset,
                 ep->mailbox->attr.smsg_attr.mbox_maxcredit, ep->mailbox->attr.smsg_attr.msg_maxsize));

    grc = GNI_SmsgInit (ep->smsg_ep_handle.gni_handle, &ep->mailbox->attr.smsg_attr,
                        &ep->remote_attr->smsg_attr);
    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
        BTL_ERROR(("error initializing SMSG protocol. rc = %d", grc));

        return mca_btl_rc_ugni_to_opal (grc);
    }

    /* set the local event data to the local index and the remote event data to my
     * index on the remote peer. This makes lookup of endpoints on completion take
     * a single lookup in the endpoints array. we will not be able to change the
     * remote peer's index in the endpoint's array after this point. */
    GNI_EpSetEventData (ep->smsg_ep_handle.gni_handle, ep->index, ep->remote_attr->index);

    ep->rmt_irq_mem_hndl = ep->remote_attr->rmt_irq_mem_hndl;
    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTED;
    (void) opal_atomic_add_fetch_32 (&ep->smsg_ep_handle.device->smsg_connections, 1);

    /* send all pending messages */
    BTL_VERBOSE(("endpoint connected. posting %u sends", (unsigned int) opal_list_get_size (&ep->frag_wait_list)));

    rc = mca_btl_ugni_progress_send_wait_list (ep);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
        if (false == ep->wait_listed) {
            opal_list_append (&ugni_module->ep_wait_list, &ep->super);
            ep->wait_listed = true;
        }
        OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
    }

    free (ep->remote_attr);
    ep->remote_attr = NULL;

    return OPAL_SUCCESS;
}

static inline int mca_btl_scif_ep_connect_start_active (mca_btl_base_endpoint_t *ep) {
    int rc = OPAL_SUCCESS;

    BTL_VERBOSE(("initiaiting connection to remote peer %d with port: %u on local scif node: %u",
                 ep->peer_proc->proc_name.vpid, ep->port_id.port, ep->port_id.node));

    opal_mutex_lock (&ep->lock);
    do {
        if (MCA_BTL_SCIF_EP_STATE_INIT != ep->state) {
            /* the accept thread has already finished this connection */
            rc = OPAL_SUCCESS;
            break;
        }

        ep->state = MCA_BTL_SCIF_EP_STATE_CONNECTING;

        ep->scif_epd = scif_open ();
        if (OPAL_UNLIKELY(SCIF_OPEN_FAILED == ep->scif_epd)) {
            BTL_VERBOSE(("error creating new scif endpoint"));
            rc = OPAL_ERROR;
            break;
        }

        rc = scif_connect (ep->scif_epd, &ep->port_id);
        if (OPAL_UNLIKELY(-1 == rc)) {
            /* the connection attempt failed. this could mean the peer is currently
             * processing connections. we will to try again later. */
            BTL_VERBOSE(("error connecting to scif peer. %d", errno));
            rc = OPAL_ERR_RESOURCE_BUSY;
            break;
        }

        rc = scif_send (ep->scif_epd, &OPAL_PROC_MY_NAME, sizeof (OPAL_PROC_MY_NAME), SCIF_SEND_BLOCK);
        if (OPAL_UNLIKELY(-1 == rc)) {
            BTL_VERBOSE(("error in scif_send"));
            rc = OPAL_ERROR;
            break;
        }

        /* build connection data */
        rc = mca_btl_scif_ep_connect_finish (ep, false);
    } while (0);

    if (OPAL_SUCCESS != rc) {
        scif_close (ep->scif_epd);
        ep->scif_epd = -1;
        ep->state = MCA_BTL_SCIF_EP_STATE_INIT;
    }

    opal_mutex_unlock (&ep->lock);

    return rc;
}

/*
 * Connect function.  Start initiation of connections to a remote
 * peer.  We send our Queue Pair information over the RML/OOB
 * communication mechanism.  On completion of our send, a send
 * completion handler is called.
 */
static int xoob_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc,
                                     mca_btl_base_endpoint_t *endpoint)
{
    int rc = OMPI_SUCCESS;

    OPAL_THREAD_LOCK(&endpoint->ib_addr->addr_lock);
    switch (endpoint->ib_addr->status) {
        case MCA_BTL_IB_ADDR_CLOSED:
            BTL_VERBOSE(("The IB addr: sid %" PRIx64 " lid %d"
                        "in MCA_BTL_IB_ADDR_CLOSED status,"
                        " sending ENDPOINT_XOOB_CONNECT_REQUEST\n",
                        endpoint->ib_addr->subnet_id,endpoint->ib_addr->lid));
            if (OMPI_SUCCESS != (rc = xoob_send_qp_create(endpoint))) {
                break;
            }

            /* Send connection info over to remote endpoint */
            endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
            endpoint->ib_addr->status = MCA_BTL_IB_ADDR_CONNECTING;
            if (OMPI_SUCCESS !=
                    (rc = xoob_send_connect_data(endpoint, ENDPOINT_XOOB_CONNECT_REQUEST))) {
                BTL_ERROR(("Error sending connect request, error code %d", rc));
            }
            break;
        case MCA_BTL_IB_ADDR_CONNECTING:
            BTL_VERBOSE(("The IB addr: sid %" PRIx64 " lid %d"
                        "in MCA_BTL_IB_ADDR_CONNECTING status,"
                        " Subscribing to this address\n",
                        endpoint->ib_addr->subnet_id,endpoint->ib_addr->lid));
            /* some body already connectng to this machine, lets wait */
            opal_list_append(&endpoint->ib_addr->pending_ep, &(endpoint->super));
            endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
            break;
        case MCA_BTL_IB_ADDR_CONNECTED:
            /* so we have the send qp, we just need the recive site.
             * Send request for SRQ numbers */
            BTL_VERBOSE(("The IB addr: sid %" PRIx64 " lid %d"
                        "in MCA_BTL_IB_ADDR_CONNECTED status,"
                        " sending ENDPOINT_XOOB_CONNECT_XRC_REQUEST\n",
                        endpoint->ib_addr->subnet_id,endpoint->ib_addr->lid));
            endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
            if (OMPI_SUCCESS !=
                    (rc = xoob_send_connect_data(endpoint, ENDPOINT_XOOB_CONNECT_XRC_REQUEST))) {
                BTL_ERROR(("error sending xrc connect request, error code %d", rc));
            }
            break;
        default :
            BTL_ERROR(("Invalid endpoint status %d", endpoint->ib_addr->status));
    }
    OPAL_THREAD_UNLOCK(&endpoint->ib_addr->addr_lock);
    return rc;
}

/**
 * Initiate a get operation.
 *
 * @param btl (IN)         BTL module
 * @param endpoint (IN)    BTL addressing information
 * @param descriptor (IN)  Description of the data to be transferred
 */
int mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
                      struct mca_btl_base_endpoint_t *endpoint,
                      struct mca_btl_base_descriptor_t *des) {
    mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) des;
    mca_btl_ugni_segment_t *src_seg = (mca_btl_ugni_segment_t *) des->des_remote;
    mca_btl_ugni_segment_t *dst_seg = (mca_btl_ugni_segment_t *) des->des_local;
    size_t size = src_seg->base.seg_len - src_seg->extra_byte_count;
    bool check;

    BTL_VERBOSE(("Using RDMA/FMA Get"));

    /* cause endpoint to bind if it isn't already (bind is sufficient for rdma) */
    (void) mca_btl_ugni_check_endpoint_state(endpoint);

    /* Check if the get is aligned/sized on a multiple of 4 */
    check = !!((des->des_remote->seg_addr.lval | des->des_local->seg_addr.lval | size) & 3);

    if (OPAL_UNLIKELY(check || size > mca_btl_ugni_component.ugni_get_limit)) {
        /* switch to put */
        return OPAL_ERR_NOT_AVAILABLE;
    }

    if (src_seg->extra_byte_count) {
        memmove ((char *) dst_seg->base.seg_addr.pval + size, src_seg->extra_bytes, src_seg->extra_byte_count);
        src_seg->base.seg_len = size;
        dst_seg->base.seg_len = size;
    }

    des->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;

    return mca_btl_ugni_post (frag, true, dst_seg, src_seg);
}

int mca_btl_ugni_ep_connect_progress (mca_btl_base_endpoint_t *ep) {
    int rc;

    BTL_VERBOSE(("progressing connection for endpoint %p with state %d", (void *)ep, ep->state));

    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
        return OPAL_SUCCESS;
    }

    if (MCA_BTL_UGNI_EP_STATE_RDMA >= ep->state) {
        rc = mca_btl_ugni_ep_connect_start (ep);
        if (OPAL_SUCCESS != rc) {
            return rc;
        }
    }

    if (GNI_SMSG_TYPE_INVALID == ep->remote_attr.smsg_attr.msg_type) {
        /* use datagram to exchange connection information with the remote peer */
        rc = mca_btl_ugni_directed_ep_post (ep);
        if (OPAL_SUCCESS == rc) {
            rc = OPAL_ERR_RESOURCE_BUSY;
        }
        return rc;
    }

    return mca_btl_ugni_ep_connect_finish (ep);
}

static int mca_btl_ud_modex_send(void)
{
    int rc;
    size_t i;
    size_t size;
    mca_btl_ud_addr_t* addrs = NULL;

    size = mca_btl_ofud_component.num_btls * sizeof(mca_btl_ud_addr_t);
    if(size != 0) {
        addrs = (mca_btl_ud_addr_t*)malloc(size);
        if(NULL == addrs) {
            return OMPI_ERR_OUT_OF_RESOURCE;
        }

        for(i = 0; i < mca_btl_ofud_component.num_btls; i++) {
            mca_btl_ud_module_t* btl = &mca_btl_ofud_component.ud_btls[i];
            addrs[i] = btl->addr;
    
            BTL_VERBOSE((0, "modex_send QP num %x, LID = %x",
              addrs[i].qp_num, addrs[i].lid));
        }
    }

    rc = ompi_modex_send(
            &mca_btl_ofud_component.super.btl_version, addrs, size);
    if(NULL != addrs) {
        free(addrs);
    }
    return rc;
}

int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnect) {
    gni_return_t rc;

    do {
        if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
            /* nothing to do */
            break;
        }

        if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state && send_disconnect) {
            rc = GNI_SmsgSendWTag (ep->smsg_ep_handle, NULL, 0, NULL, 0, -1,
                                   MCA_BTL_UGNI_TAG_DISCONNECT);
            if (GNI_RC_SUCCESS != rc) {
                BTL_VERBOSE(("btl/ugni could not send close message"));
            }

            /* we might want to wait for local completion here (do we even care) */
        }

        (void) ompi_common_ugni_ep_destroy (&ep->smsg_ep_handle);
        (void) ompi_common_ugni_ep_destroy (&ep->rdma_ep_handle);

        OMPI_FREE_LIST_RETURN(&ep->btl->smsg_mboxes, ((ompi_free_list_item_t *) ep->mailbox));
        ep->mailbox = NULL;

        ep->state = MCA_BTL_UGNI_EP_STATE_INIT;
    } while (0);

    return OMPI_SUCCESS;
}

/**
 * Prepare the dst buffer
 *
 * @param btl (IN)      BTL module
 * @param peer (IN)     BTL peer addressing
 * prepare dest's behavior depends on the following: 
 * Has a valid memory registration been passed to prepare_src? 
 *    if so we attempt to use the pre-registred user-buffer, if the memory registration 
 *    is to small (only a portion of the user buffer) then we must reregister the user buffer 
 * Has the user requested the memory to be left pinned? 
 *    if so we insert the memory registration into a memory tree for later lookup, we 
 *    may also remove a previous registration if a MRU (most recently used) list of 
 *    registions is full, this prevents resources from being exhausted.
 */
mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* endpoint,
    mca_mpool_base_registration_t* registration,
    struct ompi_convertor_t* convertor,
    size_t reserve,
    size_t* size)
{
    mca_btl_openib_module_t *openib_btl;
    mca_btl_openib_frag_t *frag;
    mca_btl_openib_reg_t *openib_reg;
    int rc;
    ptrdiff_t lb;

    openib_btl = (mca_btl_openib_module_t*)btl;
    
    MCA_BTL_IB_FRAG_ALLOC_RECV_FRAG(btl, frag, rc);
    if(NULL == frag) {
        return NULL;
    }
    
    ompi_ddt_type_lb(convertor->pDesc, &lb);
    frag->segment.seg_addr.pval = convertor->pBaseBuf + lb +
        convertor->bConverted;

    if(NULL == registration){
        /* we didn't get a memory registration passed in, so we have to
         * register the region ourselves
         */ 
        rc = btl->btl_mpool->mpool_register(btl->btl_mpool,
                frag->segment.seg_addr.pval, *size, 0, &registration);
        if(OMPI_SUCCESS != rc || NULL == registration) {
            MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
            return NULL;
        }
        /* keep track of the registration we did */
        frag->registration = (mca_btl_openib_reg_t*)registration;
    }
    openib_reg = (mca_btl_openib_reg_t*)registration;

    frag->sg_entry.length = *size;
    frag->sg_entry.lkey = openib_reg->mr->lkey;
    frag->sg_entry.addr = (unsigned long) frag->segment.seg_addr.pval;

    frag->segment.seg_len = *size;
    frag->segment.seg_key.key32[0] = openib_reg->mr->rkey;

    frag->base.des_dst = &frag->segment;
    frag->base.des_dst_cnt = 1;
    frag->base.des_src = NULL;
    frag->base.des_src_cnt = 0;
    frag->base.des_flags = 0;

    BTL_VERBOSE(("frag->sg_entry.lkey = %lu .addr = %llu "
                "frag->segment.seg_key.key32[0] = %lu",
                frag->sg_entry.lkey, frag->sg_entry.addr,
                frag->segment.seg_key.key32[0]));

    return &frag->base;
}

/*
 * Look for an existing TCP process instance based on the globally unique
 * process identifier.
 */
mca_btl_tcp_proc_t* mca_btl_tcp_proc_lookup(const opal_process_name_t *name)
{
    mca_btl_tcp_proc_t* proc = NULL;

    OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
    opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs,
                              *name, (void**)&proc);
    OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
    if (OPAL_UNLIKELY(NULL == proc)) {
        mca_btl_base_endpoint_t *endpoint;
        opal_proc_t *opal_proc;

        BTL_VERBOSE(("adding tcp proc for unknown peer {.jobid = 0x%x, .vpid = 0x%x}",
                     name->jobid, name->vpid));

        opal_proc = opal_proc_for_name (*name);
        if (NULL == opal_proc) {
            return NULL;
        }

        /* try adding this proc to each btl until */
        for( uint32_t i = 0; i < mca_btl_tcp_component.tcp_num_btls; ++i ) {
            endpoint = NULL;
            (void) mca_btl_tcp_add_procs (&mca_btl_tcp_component.tcp_btls[i]->super, 1, &opal_proc,
                                          &endpoint, NULL);
            if (NULL != endpoint && NULL == proc) {
                /* get the proc and continue on (could probably just break here) */
                proc = endpoint->endpoint_proc;
            }
        }
    }

    return proc;
}

static int mca_btl_ugni_endpoint_get_modex (mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_modex_t *modex;
    size_t msg_size;
    int rc;

    assert (NULL != ep && NULL != ep->peer_proc);

    /* Receive the modex */
    OPAL_MODEX_RECV(rc, &mca_btl_ugni_component.super.btl_version,
                    &ep->peer_proc->proc_name, (void **)&modex, &msg_size);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        BTL_ERROR(("error receiving modex"));
        return rc;
    }

    ep->ep_rem_addr = modex->addr;
    ep->ep_rem_id = modex->id;


    BTL_VERBOSE(("received modex for ep %p. addr: %d, id: %d",  (void*)ep, ep->ep_rem_addr, ep->ep_rem_id));

    free (modex);

    return OPAL_SUCCESS;
}

/*
 * Reply to a `start - connect' message
 */
static int reply_start_connect(mca_btl_openib_endpoint_t *endpoint,
                               mca_btl_openib_rem_info_t *rem_info)
{
    int rc;

    BTL_VERBOSE(("Initialized QPs, LID = %d",
                 ((mca_btl_openib_module_t*)endpoint->endpoint_btl)->lid));

    /* Create local QP's and post receive resources */
    if (OMPI_SUCCESS != (rc = qp_create_all(endpoint))) {
        return rc;
    }

    /* Set the remote side info */
    set_remote_info(endpoint, rem_info);
    
    /* Connect to remote endpoint qp's */
    if (OMPI_SUCCESS != (rc = qp_connect_all(endpoint))) {
        return rc;
    }

    /* Send connection info over to remote endpoint */
    endpoint->endpoint_state = MCA_BTL_IB_CONNECT_ACK;
    if (OMPI_SUCCESS !=
        (rc = send_connect_data(endpoint, ENDPOINT_CONNECT_RESPONSE))) {
        BTL_ERROR(("error in endpoint send connect request error code is %d",
                   rc));
        return rc;
    }
    return OMPI_SUCCESS;
}

static inline int
mca_btl_ugni_handle_remote_smsg_overrun (mca_btl_ugni_module_t *btl)
{
    gni_cq_entry_t event_data;
    unsigned int ep_index;
    int count, rc;

    BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. "
                 "processing message backlog..."));

    /* we don't know which endpoint lost an smsg completion. clear the
       smsg remote cq and check all mailboxes */

    /* clear out remote cq */
    do {
        rc = GNI_CqGetEvent (btl->smsg_remote_cq, &event_data);
    } while (GNI_RC_NOT_DONE != rc);

    for (ep_index = 0, count = 0 ; ep_index < btl->endpoint_count ; ++ep_index) {
        mca_btl_base_endpoint_t *ep = btl->endpoints[ep_index];

        if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) {
            continue;
        }

        /* clear out smsg mailbox */
        rc = mca_btl_ugni_smsg_process (ep);
        if (OPAL_LIKELY(rc >= 0)) {
            count += rc;
        }
    }

    return count;
}

void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
{
    struct ibv_qp_init_attr qp_init_attr;
    struct ibv_qp_attr attr;
    enum ibv_qp_attr_mask mask = 0;
    struct mca_btl_openib_module_t *btl;

    BTL_VERBOSE(("APM XRC: Loading alternative path"));
    assert (NULL != ep);
    btl = ep->endpoint_btl;

    if (ibv_query_xrc_rcv_qp(btl->device->xrc_domain, qp_num, &attr, mask, &qp_init_attr))
        BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp_num));

    if (mca_btl_openib_component.apm_lmc &&
            attr.ah_attr.src_path_bits - btl->src_path_bits < mca_btl_openib_component.apm_lmc) {
        apm_update_attr(&attr, &mask);
    } else {
        if (mca_btl_openib_component.apm_ports) {
            /* Try to migrate to next port */
            if (OPAL_SUCCESS != apm_update_port(ep, &attr, &mask))
                return;
        } else {
            BTL_ERROR(("Failed to load alternative path, all %d were used",
                        attr.ah_attr.src_path_bits - btl->src_path_bits));
        }
    }

    ibv_modify_xrc_rcv_qp(btl->device->xrc_domain, qp_num, &attr, mask);
    /* Maybe the qp already was modified by other process - ignoring error */
}

/* Find endpoint for specific subnet/lid/message */
static mca_btl_openib_endpoint_t* xoob_find_endpoint(ompi_process_name_t* process_name,
        uint64_t subnet_id, uint16_t lid, uint8_t message_type)
{
    size_t i;
    mca_btl_openib_proc_t *ib_proc;
    mca_btl_openib_endpoint_t *ib_endpoint = NULL;
    bool found = false;

    BTL_VERBOSE(("Searching for ep and proc with follow parameters:"
                "jobid %d, vpid %d, "
                "sid %" PRIx64 ", lid %d",
                process_name->jobid, process_name->vpid,
                subnet_id, lid));


    /* find ibproc */
    OPAL_THREAD_LOCK(&mca_btl_openib_component.ib_lock);
    for (ib_proc = (mca_btl_openib_proc_t*)
            opal_list_get_first(&mca_btl_openib_component.ib_procs);
            ib_proc != (mca_btl_openib_proc_t*)
            opal_list_get_end(&mca_btl_openib_component.ib_procs);
            ib_proc  = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) {
        if (OPAL_EQUAL == ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
                    &ib_proc->proc_ompi->proc_name, process_name)) {
            found = true;
            break;
        }
    }
    /* we found our ib_proc, lets find endpoint now */
    if (found) {
        for (i = 0; i < ib_proc->proc_endpoint_count; i++) {
            ib_endpoint = ib_proc->proc_endpoints[i];
            /* we need to check different
             * lid for different message type */
            if (ENDPOINT_XOOB_CONNECT_RESPONSE == message_type ||
                    ENDPOINT_XOOB_CONNECT_XRC_RESPONSE == message_type) {
                /* response message */
                if (ib_endpoint->subnet_id == subnet_id &&
                        ib_endpoint->ib_addr->lid == lid) {
                    break; /* Found one */
                }
            } else {
                /* request message */
                if (ib_endpoint->subnet_id == subnet_id &&
                        ib_endpoint->endpoint_btl->lid == lid) {
                    break; /* Found one */
                }
            }
        }
        if (NULL == ib_endpoint) {
                BTL_ERROR(("can't find suitable endpoint for this peer\n"));
        }
    } else {
            BTL_ERROR(("can't find suitable endpoint for this peer\n"));
    }
    OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock);
    return ib_endpoint;
}

int mca_btl_scif_module_init (void)
{
    int rc;

    /* create an endpoint to listen for connections */
    mca_btl_scif_module.scif_fd = scif_open ();
    if (-1 == mca_btl_scif_module.scif_fd) {
        BTL_VERBOSE(("scif_open failed. errno = %d", errno));
        return OPAL_ERROR;
    }

    /* bind the endpoint to a port */
    mca_btl_scif_module.port_id.port = scif_bind (mca_btl_scif_module.scif_fd, 0);
    if (-1 == mca_btl_scif_module.port_id.port) {
        BTL_VERBOSE(("scif_bind failed. errno = %d", errno));
        scif_close (mca_btl_scif_module.scif_fd);
        mca_btl_scif_module.scif_fd = -1;
        return OPAL_ERROR;
    }

    /* determine this processes node id */
    rc = scif_get_nodeIDs (NULL, 0, &mca_btl_scif_module.port_id.node);
    if (-1 == rc) {
        BTL_VERBOSE(("btl/scif error getting node id of this node"));
        return OPAL_ERROR;
    }

    /* Listen for connections */
    /* TODO - base the maximum backlog off something */
    rc = scif_listen (mca_btl_scif_module.scif_fd, 64);
    if (-1 == rc) {
        BTL_VERBOSE(("scif_listen failed. errno = %d", errno));
        scif_close (mca_btl_scif_module.scif_fd);
        mca_btl_scif_module.scif_fd = -1;
        return OPAL_ERROR;
    }

    BTL_VERBOSE(("btl/scif: listening @ port %u on node %u\n",
                 mca_btl_scif_module.port_id.port, mca_btl_scif_module.port_id.node));

    OBJ_CONSTRUCT(&mca_btl_scif_module.dma_frags, opal_free_list_t);
    OBJ_CONSTRUCT(&mca_btl_scif_module.eager_frags, opal_free_list_t);

    return OPAL_SUCCESS;
}