int opal_hwloc_copy(hwloc_topology_t *dest, hwloc_topology_t src, opal_data_type_t type)
{
    char *xml;
    int len;
    struct hwloc_topology_support *support, *destsupport;

    if (0 != hwloc_topology_export_xmlbuffer(src, &xml, &len)) {
        return OPAL_ERROR;
    }
    if (0 != hwloc_topology_init(dest)) {
        free(xml);
        return OPAL_ERROR;
    }
    if (0 != hwloc_topology_set_xmlbuffer(*dest, xml, len)) {
        hwloc_topology_destroy(*dest);
        free(xml);
        return OPAL_ERROR;
    }
    if (0 != hwloc_topology_load(*dest)) {
        hwloc_topology_destroy(*dest);
        free(xml);
        return OPAL_ERROR;
    }
    free(xml);

    /* get the available support - hwloc unfortunately does
     * not include this info in its xml support!
     */
    support = (struct hwloc_topology_support*)hwloc_topology_get_support(src);
    destsupport = (struct hwloc_topology_support*)hwloc_topology_get_support(*dest);
    *destsupport = *support;

    return OPAL_SUCCESS;
}

static int _get_cpuinfo(uint32_t *nsockets, uint32_t *ncores,
			uint32_t *nthreads, uint32_t *npus)
{
	hwloc_topology_t topology;

	if (hwloc_topology_init(&topology)) {
		/* error in initialize hwloc library */
		error("%s: hwloc_topology_init() failed", __func__);
		return -1;
	}
	/* parse full system info */
	hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM);
	/* ignores cache, misc */
	hwloc_topology_ignore_type (topology, HWLOC_OBJ_CACHE);
	hwloc_topology_ignore_type (topology, HWLOC_OBJ_MISC);
	/* load topology */
	if (hwloc_topology_load(topology)) {
		error("%s: hwloc_topology_load() failed", __func__);
		hwloc_topology_destroy(topology);
		return -1;
	}

	*nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
							HWLOC_OBJ_SOCKET);
	*ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
						      HWLOC_OBJ_CORE);
	*nthreads = (uint32_t) hwloc_get_nbobjs_by_type(topology,
							HWLOC_OBJ_PU);
	*npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
						    HWLOC_OBJ_PU);
	hwloc_topology_destroy(topology);
	return 0;
}

int main(void)
{
  static hwloc_topology_t oldtopology, topology;
  hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
  unsigned node_indexes[3], core_indexes[6];
  float node_distances[9], core_distances[36];
  unsigned i,j;
  int err;

  for(i=0; i<3; i++) {
    node_indexes[i] = i;
    for(j=0; j<3; j++)
      node_distances[i*3+j] = (i == j ? 10.f : 20.f);
  }
  for(i=0; i<6; i++) {
    core_indexes[i] = i;
    for(j=0; j<6; j++)
      core_distances[i*6+j] = (i == j ? 4.f : 8.f);
  }

  hwloc_topology_init(&oldtopology);
  printf("building fake 'node:3 core:2 pu:4' topology\n");
  hwloc_topology_set_synthetic(oldtopology, "node:3 core:2 pu:4");
  printf("adding node and core matrices\n");
  hwloc_topology_set_distance_matrix(oldtopology, HWLOC_OBJ_NUMANODE, 3, node_indexes, node_distances);
  hwloc_topology_set_distance_matrix(oldtopology, HWLOC_OBJ_CORE, 6, core_indexes, core_distances);
  hwloc_topology_load(oldtopology);

  printf("duplicating\n");
  hwloc_topology_dup(&topology, oldtopology);
  printf("destroying the old topology\n");
  hwloc_topology_destroy(oldtopology);

  /* remove the entire third node */
  printf("removing one node\n");
  hwloc_bitmap_fill(cpuset);
  hwloc_bitmap_clr_range(cpuset, 16, 23);
  err = hwloc_topology_restrict(topology, cpuset, HWLOC_RESTRICT_FLAG_ADAPT_DISTANCES);
  assert(!err);
  printf("checking the result\n");
  assert(hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE) == 2);

  hwloc_topology_destroy(topology);

  hwloc_bitmap_free(cpuset);

  return 0;
}

void Hwloc::unloadHwloc ()
{
#ifdef HWLOC
   /* Destroy topology object. */
   hwloc_topology_destroy( _hwlocTopology );
#endif
}

/*
 * Get the number of net cards in a NUMA node
 */
int hw_get_nnetcards(int NUMAnode)
{
  int count, net_NUMAnode, error;
  hwloc_obj_t obj, obj_anc;

 count = 0;
 hwloc_topology_t topo_net;
 error = hwloc_topology_init(&topo_net);
 hwloc_topology_set_flags(topo_net, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
 if (!error){
    hwloc_topology_load(topo_net);
    for (obj = hwloc_get_obj_by_type(topo_net, HWLOC_OBJ_OS_DEVICE, 0);
         obj;
         obj = hwloc_get_next_osdev(topo_net,obj))
         if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK ||
             obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS){
             obj_anc = hwloc_get_non_io_ancestor_obj(topo_net,obj);
             net_NUMAnode = hwloc_bitmap_first(obj_anc->nodeset);
             //only if the MPI NUMA node is equal to the found net card
               if(NUMAnode == net_NUMAnode)
                 count++;
         }        
  }         
  hwloc_topology_destroy(topo_net);

  return count;
}

int main(void)
{
  hwloc_topology_t topology;
  int i;
  int err;

  hwloc_topology_init(&topology);
  hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
  hwloc_topology_load(topology);

  for(i=0; ; i++) {
    hwloc_bitmap_t set;
    hwloc_obj_t osdev, ancestor;
    const char *value;

    osdev = hwloc_intel_mic_get_device_osdev_by_index(topology, i);
    if (!osdev)
      break;
    assert(osdev);

    ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);

    printf("found OSDev %s\n", osdev->name);
    err = strncmp(osdev->name, "mic", 3);
    assert(!err);
    assert(atoi(osdev->name+3) == (int) i);

    assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);

    value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
    err = strcmp(value, "MIC");
    assert(!err);

    value = hwloc_obj_get_info_by_name(osdev, "MICFamily");
    printf("found MICFamily %s\n", value);
    value = hwloc_obj_get_info_by_name(osdev, "MICSKU");
    printf("found MICSKU %s\n", value);
    value = hwloc_obj_get_info_by_name(osdev, "MICActiveCores");
    printf("found MICActiveCores %s\n", value);
    value = hwloc_obj_get_info_by_name(osdev, "MICMemorySize");
    printf("found MICMemorySize %s\n", value);

    set = hwloc_bitmap_alloc();
    err = hwloc_intel_mic_get_device_cpuset(topology, i, set);
    if (err < 0) {
      printf("failed to get cpuset for device %d\n", i);
    } else {
      char *cpuset_string = NULL;
      hwloc_bitmap_asprintf(&cpuset_string, set);
      printf("got cpuset %s for device %d\n", cpuset_string, i);
      assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
      free(cpuset_string);
    }
    hwloc_bitmap_free(set);
  }

  hwloc_topology_destroy(topology);

  return 0;
}

void chpl_topo_exit(void) {
  if (!haveTopology) {
    return;
  }

  hwloc_topology_destroy(topology);
}

int rsreader_hwloc_load (resrc_api_ctx_t *rsapi, const char *buf, size_t len,
        uint32_t rank, rsreader_t r_mode, machs_t *machs, char **err_str)
{
    int rc = -1;
    rssig_t *sig = NULL;
    hwloc_topology_t topo;

    if (!machs)
        goto done;

    if (hwloc_topology_init (&topo) != 0)
        goto done;
    if (hwloc_topology_set_xmlbuffer (topo, buf, len) != 0)
        goto err;
    if (hwloc_topology_load (topo) != 0)
        goto err;
    if (rs2rank_set_signature ((char*)buf, len, topo, &sig) != 0)
        goto err;
    if (rs2rank_tab_update (machs, get_hn (topo), sig, rank) != 0)
        goto err;

    if (r_mode == RSREADER_HWLOC) {
        const char *s = rs2rank_get_digest (sig);
        if (!resrc_generate_hwloc_resources (rsapi, topo, s, err_str))
            goto err;
    }

    rc = 0;
err:
    hwloc_topology_destroy (topo);
done:
    return rc;
}

void terminate_thread_pool(){
  int id;
  int *ret=NULL;
  work_t work;

  if(pool){
    work.task=NULL;
    for (id=0;id<pool->nb_threads;id++){
      submit_work(&work,id);
    }


    for (id=0;id<pool->nb_threads;id++){
      pthread_join(pool->thread_list[id],(void **) &ret);
      FREE(ret);
      pthread_cond_destroy(pool->cond_var +id);
      pthread_mutex_destroy(pool->list_lock +id);
      if (pool->working_list[id].next != NULL)
	if(verbose_level >= WARNING)
	  printf("Working list of thread %d not empty!\n",id);
    }

    hwloc_topology_destroy(pool->topology);
    FREE(pool -> thread_list);
    FREE(pool -> working_list);
    FREE(pool -> cond_var);
    FREE(pool -> list_lock);
    FREE(pool -> local);
    FREE(pool);
    pool = NULL;
  }
}

/* Wrapper routines for hwloc */
void qrm_hwloc_bind(int id)
{
  int depth, ret;
  unsigned i, n;
  int topodepth;
  hwloc_topology_t topology;
  hwloc_cpuset_t cpuset;
  hwloc_obj_t obj;
  
  hwloc_topology_init(&topology);
  
  hwloc_topology_load(topology);
  
  obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_CORE, id); 
  
  ret = hwloc_set_cpubind(topology, obj->cpuset, HWLOC_CPUBIND_THREAD);
  if (ret) {
    printf("Couldn't bind to core %d\n", id); 
    assert(0);
  } else {
    printf("Bound to core %d\n", id); 
  }    
  
  hwloc_topology_destroy(topology);
  
  return;
}

JNIEXPORT void JNICALL Java_com_rr_core_os_NativeHooksImpl_jniSetPriority( JNIEnv *env, jclass clazz, jint cpumask, jint priority ) {

    int topodepth;
    hwloc_topology_t topology;
    hwloc_cpuset_t cpuset;

    hwloc_topology_init(&topology);
    hwloc_topology_load(topology);
    topodepth = hwloc_topology_get_depth(topology);

    cpuset = hwloc_bitmap_alloc();
    hwloc_bitmap_from_ulong( cpuset, (unsigned int)cpumask );

    char *str;
    hwloc_bitmap_asprintf(&str, cpuset);

    printf("cpumask [%d] => hwloc [%s]\n", cpumask, str);

    if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD)) {
        printf("Couldn't bind cpuset %s\n", str);
    } else {
        printf("BOUND cpuset %s\n", str);
    }

    free(str);

    /* Free our cpuset copy */
    hwloc_bitmap_free(cpuset);

    /* Destroy topology object. */
    hwloc_topology_destroy(topology);
}

int main(void)
{
  hwloc_topology_t topology;
  hwloc_bitmap_t cpuset;
  int err;

  /* check the OS topology */
  hwloc_topology_init(&topology);
  hwloc_topology_load(topology);
  assert(hwloc_topology_is_thissystem(topology));

  cpuset = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset(topology));
  result("Binding with OS backend", hwloc_set_cpubind(topology, cpuset, 0));

  hwloc_topology_destroy(topology);

  /* We're assume there is a real processor numbered 0 */
  hwloc_bitmap_zero(cpuset);
  hwloc_bitmap_set(cpuset, 0);

  /* check a synthetic topology */
  hwloc_topology_init(&topology);
  hwloc_topology_set_synthetic(topology, "1");
  hwloc_topology_load(topology);
  assert(!hwloc_topology_is_thissystem(topology));

  err = hwloc_set_cpubind(topology, cpuset, 0);
  result("Binding with synthetic backend", err);
  assert(!err);

  hwloc_topology_destroy(topology);

  /* check a synthetic topology but assuming it's the system topology */
  hwloc_topology_init(&topology);
  hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
  hwloc_topology_set_synthetic(topology, "1");
  hwloc_topology_load(topology);
  assert(hwloc_topology_is_thissystem(topology));

  result("Binding with synthetic backend faking is_thissystem", hwloc_set_cpubind(topology, cpuset, 0));

  hwloc_topology_destroy(topology);

  hwloc_bitmap_free(cpuset);

  return 0;
}

static int adopt(int fd, unsigned long fileoffset, unsigned long mmap_address, unsigned long mmap_length, int synthetic_with_distances)
{
  static hwloc_topology_t adopted;
  char *xmlbuf;
  int xmlbuflen;
  char *origxmlbuf;
  struct hwloc_distances_s *distances;
  unsigned nr = 1;
  int err;
  int ret = EXIT_SKIP;

  err = lseek(fd, 0, SEEK_SET);
  assert(!err);

  printf(" reading XML dump\n");
  origxmlbuf = malloc(fileoffset);
  assert(origxmlbuf);
  err = read(fd, origxmlbuf, fileoffset);
  assert(err > 0);

  printf(" adopting from file at offset %lu with addr %lx len %lu\n", fileoffset, mmap_address, mmap_length);

  err = hwloc_shmem_topology_adopt(&adopted, fd, fileoffset, (void*)(uintptr_t)mmap_address, mmap_length, 0);
  if (err == -1 && errno == EBUSY) {
    /* may fail on 32bits and on some OS (e.g. darwin from time to time), and even on Linux/64bits if unlucky */
    fprintf(stderr, "Failed to shmem adopt, requested mapping is busy\n");
    goto out_with_origxmlbuf;
  }
  assert(!err);
  printf(" adopted OK\n");

  err = hwloc_distances_get_by_type(adopted, HWLOC_OBJ_NUMANODE, &nr, &distances, 0, 0);
  assert(!err);
  if (synthetic_with_distances) {
    assert(nr == 1);
    assert(distances->nbobjs == 3);
    assert(distances->kind == (HWLOC_DISTANCES_KIND_MEANS_LATENCY|HWLOC_DISTANCES_KIND_FROM_USER));
    hwloc_distances_release(adopted, distances);
    printf(" distances OK\n");
  }

  err = hwloc_topology_export_xmlbuffer(adopted, &xmlbuf, &xmlbuflen, 0);
  assert(!err);
  printf(" XML export %d bytes\n", xmlbuflen);
  assert((unsigned long) xmlbuflen < fileoffset);
  assert(!memcmp(origxmlbuf, xmlbuf, xmlbuflen));
  hwloc_free_xmlbuffer(adopted, xmlbuf);
  printf(" XML export is identical to original\n");

  hwloc_topology_destroy(adopted);
  printf(" destroyed\n");

  ret = EXIT_SUCCESS;

 out_with_origxmlbuf:
  free(origxmlbuf);
  return ret;
}

int main(void)
{
  hwloc_topology_t topology;
  hwloc_obj_t obj;

  hwloc_topology_init(&topology);
  hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_ALL);
  hwloc_topology_load(topology);

  printf("Found %d bridges\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_BRIDGE));
  obj = NULL;
  while ((obj = hwloc_get_next_bridge(topology, obj)) != NULL) {
    assert(obj->type == HWLOC_OBJ_BRIDGE);
    /* only host->pci and pci->pci bridge supported so far */
    if (obj->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_HOST) {
      assert(obj->attr->bridge.downstream_type == HWLOC_OBJ_BRIDGE_PCI);
      printf(" Found host->PCI bridge for domain %04x bus %02x-%02x\n",
	     obj->attr->bridge.downstream.pci.domain,
	     obj->attr->bridge.downstream.pci.secondary_bus,
	     obj->attr->bridge.downstream.pci.subordinate_bus);
    } else {
      assert(obj->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_PCI);
      assert(obj->attr->bridge.downstream_type == HWLOC_OBJ_BRIDGE_PCI);
      printf(" Found PCI->PCI bridge [%04x:%04x] for domain %04x bus %02x-%02x\n",
	     obj->attr->bridge.upstream.pci.vendor_id,
	     obj->attr->bridge.upstream.pci.device_id,
	     obj->attr->bridge.downstream.pci.domain,
	     obj->attr->bridge.downstream.pci.secondary_bus,
	     obj->attr->bridge.downstream.pci.subordinate_bus);
    }
  }

  printf("Found %d PCI devices\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PCI_DEVICE));
  obj = NULL;
  while ((obj = hwloc_get_next_pcidev(topology, obj)) != NULL) {
    assert(obj->type == HWLOC_OBJ_PCI_DEVICE);
    printf(" Found PCI device class %04x vendor %04x model %04x\n",
	   obj->attr->pcidev.class_id, obj->attr->pcidev.vendor_id, obj->attr->pcidev.device_id);
  }

  printf("Found %d OS devices\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE));
  obj = NULL;
  while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
    assert(obj->type == HWLOC_OBJ_OS_DEVICE);
    printf(" Found OS device %s subtype %d\n", obj->name, obj->attr->osdev.type);
  }

  assert(HWLOC_TYPE_DEPTH_BRIDGE == hwloc_get_type_depth(topology, HWLOC_OBJ_BRIDGE));
  assert(HWLOC_TYPE_DEPTH_PCI_DEVICE == hwloc_get_type_depth(topology, HWLOC_OBJ_PCI_DEVICE));
  assert(HWLOC_TYPE_DEPTH_OS_DEVICE == hwloc_get_type_depth(topology, HWLOC_OBJ_OS_DEVICE));
  assert(hwloc_compare_types(HWLOC_OBJ_BRIDGE, HWLOC_OBJ_PCI_DEVICE) < 0);
  assert(hwloc_compare_types(HWLOC_OBJ_BRIDGE, HWLOC_OBJ_OS_DEVICE) < 0);
  assert(hwloc_compare_types(HWLOC_OBJ_PCI_DEVICE, HWLOC_OBJ_OS_DEVICE) < 0);

  hwloc_topology_destroy(topology);

  return 0;
}

int
main (void)
{
  hwloc_topology_t topology;
  unsigned depth;
  hwloc_obj_t last;
  hwloc_obj_t *closest;
  unsigned found;
  int err;
  unsigned numprocs;
  hwloc_obj_t ancestor;

  err = hwloc_topology_init (&topology);
  if (err)
    return EXIT_FAILURE;

  hwloc_topology_set_synthetic (topology, "2 3 4 5");

  err = hwloc_topology_load (topology);
  if (err)
    return EXIT_FAILURE;

  depth = hwloc_topology_get_depth(topology);

  /* get the last object of last level */
  numprocs =  hwloc_get_nbobjs_by_depth(topology, depth-1);
  last = hwloc_get_obj_by_depth(topology, depth-1, numprocs-1);

  /* allocate the array of closest objects */
  closest = malloc(numprocs * sizeof(*closest));
  assert(closest);

  /* get closest levels */
  found = hwloc_get_closest_objs (topology, last, closest, numprocs);
  printf("looked for %u closest entries, found %u\n", numprocs, found);
  assert(found == numprocs-1);

  /* check first found is closest */
  assert(closest[0] == hwloc_get_obj_by_depth(topology, depth-1, numprocs-5 /* arity is 5 on last level */));
  /* check some other expected positions */
  assert(closest[found-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5-1 /* last of first half */));
  assert(closest[found/2-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5-1 /* last of second third of second half */));
  assert(closest[found/2/3-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5+3*5-1 /* last of third quarter of third third of second half */));

  /* get ancestor of last and less close object */
  ancestor = hwloc_get_common_ancestor_obj(topology, last, closest[found-1]);
  assert(hwloc_obj_is_in_subtree(topology, last, ancestor));
  assert(hwloc_obj_is_in_subtree(topology, closest[found-1], ancestor));
  assert(ancestor == hwloc_get_root_obj(topology)->first_child);
  printf("ancestor type %u depth %u number %u is system level\n",
	 ancestor->type, ancestor->depth, ancestor->logical_index);

  free(closest);
  hwloc_topology_destroy (topology);

  return EXIT_SUCCESS;
}

static void tdes(orte_topology_t *t)
{
    if (NULL != t->topo) {
        hwloc_topology_destroy(t->topo);
    }
    if (NULL != t->sig) {
        free(t->sig);
    }
}

int compute_context_nbr(int * ctxnbr, int threadnbr, int verbose)
{
#if (defined WITH_STARPU && defined STARPU_CONTEXT)
  if (*ctxnbr == -1)
    {
      int depth;
      unsigned i, n;
      int ncpu_per_socket, nsocket, ncpu;
      hwloc_topology_t topology;
      hwloc_obj_t obj;
      /* Allocate and initialize topology object. */
      hwloc_topology_init(&topology);
      /* ... Optionally, put detection configuration here to ignore
         some objects types, define a synthetic topology, etc....
         The default is to detect all the objects of the machine that
         the caller is allowed to access.  See Configure Topology
         Detection. */
      /* Perform the topology detection. */
      hwloc_topology_load(topology);

      depth = hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET);
      if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
        /* number of socket is unknow, let say we have quadcore... */
        ncpu_per_socket = 4;
      } else {
        ncpu    = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
        nsocket = hwloc_get_nbobjs_by_depth(topology, depth);
        ncpu_per_socket = ncpu/nsocket;
      }
      if (verbose > API_VERBOSE_NO)
        fprintf(stdout, "ncpu_per_socket %d\n", ncpu_per_socket);
      nsocket = threadnbr/ncpu_per_socket;
      if (threadnbr%ncpu_per_socket)
        nsocket ++;
      *ctxnbr = nsocket + 1;

      hwloc_topology_destroy(topology);
    }
  if (threadnbr + 1 < *ctxnbr)
    *ctxnbr = threadnbr+1;

  if (*ctxnbr == 1)
    *ctxnbr = 2;

  /* can't have more than STARPU_NMAX_SCHED_CTXS CTX */
  {
    PASTIX_INT nctx_bot = *ctxnbr - 1;
    while (*ctxnbr > STARPU_NMAX_SCHED_CTXS)
      {
        nctx_bot /= 2;
        *ctxnbr = nctx_bot + 1;
      }
  }
#endif /* WITH_STARPU */
  return 0;
}

tm_topology_t* get_local_topo_with_hwloc(void)
{
  hwloc_topology_t topology;
  tm_topology_t *res = NULL;
  hwloc_obj_t *objs = NULL;
  unsigned topodepth,depth;
  int nb_nodes,i;

  /* Build the topology */
  hwloc_topology_init(&topology);
  hwloc_topology_ignore_all_keep_structure(topology);
  hwloc_topology_load(topology);

  /* Test if symetric */
  if(!symetric(topology)){
    if(get_verbose_level() >= CRITICAL)
      fprintf(stderr,"Local toplogy not symetric!\n");
    exit(-1);
  }

  /* work on depth */
  topodepth = hwloc_topology_get_depth(topology);

  res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
  res->nb_levels = topodepth;
  res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
  res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels);
  res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);

  /* Build TreeMatch topology */
  for( depth = 0 ; depth < topodepth ; depth++ ){
    nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
    res->nb_nodes[depth] = nb_nodes;
    res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);

    objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
    objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
    hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
    res->arity[depth] = objs[0]->arity;

    /* printf("%d:",res->arity[depth]); */

    /* Build process id tab */
    for (i = 0; i < nb_nodes; i++){
      res->node_id[depth][i] = objs[i]->os_index;
      /* if(depth==topodepth-1) */
    }
    FREE(objs);
  }

  /* Destroy HWLOC topology object. */
  hwloc_topology_destroy(topology);

  /* printf("\n"); */
  return res;
}

void qrm_hwloc_topo(int *nodes, int *topo)
{
  int depth, ret;
  unsigned i, n, j, ncores, nnodes, cnode;
  int topodepth, numa;
  hwloc_topology_t topology;
  hwloc_cpuset_t cpuset;
  hwloc_obj_t obj, cobj;
  hwloc_obj_type_t otype;
  
  hwloc_topology_init(&topology);
  
  hwloc_topology_load(topology);

  /* get the number os cores and NUMA nodes */
  ncores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
  printf("ncores: %d\n",ncores);

  nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NODE);
  if(nnodes == 0){
    otype = HWLOC_OBJ_SOCKET;
    printf("grouping with sockets\n");
    nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_SOCKET);
  } else {
    otype = HWLOC_OBJ_NODE;
    printf("grouping with NUMA nodes\n");
  }

  /* get the handle for the first NUMA node */
  obj = hwloc_get_obj_by_type(topology, otype, 0); 
  
  /* get the number of cores in one NUMA node (supposedly the same for all nodes) */
  cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);
  
  for(i=0; i<nnodes; i++){
    /* get the handle for the first i-th node */
    obj = hwloc_get_obj_by_type(topology, otype, i);
    /* get the number of cores in i-th NUMA node (supposedly the same for all nodes) */
    cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);

    /* get the first core in this node */
    cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, NULL);
    topo[(i*cnode)] = cobj->logical_index;
    /* printf("%2d -- group:  %2d",i,cobj->logical_index); */
    for(j=1; j<cnode; j++){
      cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, cobj);
      topo[(i*cnode)+j] = cobj->logical_index;
      /* printf(" %2d",cobj->logical_index); */
    }
    /* printf("\n"); */
  }
  
  
  hwloc_topology_destroy(topology);
  return;
}

int main(void)
{
  hwloc_topology_t topology;
  struct ibv_device **dev_list, *dev;
  int count, i;
  int err;

  dev_list = ibv_get_device_list(&count);
  if (!dev_list) {
    fprintf(stderr, "ibv_get_device_list failed\n");
    return 0;
  }
  printf("ibv_get_device_list found %d devices\n", count);

  hwloc_topology_init(&topology);
  hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
  hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
  hwloc_topology_load(topology);

  for(i=0; i<count; i++) {
    hwloc_bitmap_t set;
    dev = dev_list[i];

    set = hwloc_bitmap_alloc();
    err = hwloc_ibv_get_device_cpuset(topology, dev, set);
    if (err < 0) {
      printf("failed to get cpuset for device %d (%s)\n",
	     i, ibv_get_device_name(dev));
    } else {
      char *cpuset_string = NULL;
      hwloc_obj_t os;

      hwloc_bitmap_asprintf(&cpuset_string, set);
      printf("got cpuset %s for device %d (%s)\n",
	     cpuset_string, i, ibv_get_device_name(dev));
      free(cpuset_string);

      os = hwloc_ibv_get_device_osdev(topology, dev);
      if (os) {
	assert(os->type == HWLOC_OBJ_OS_DEVICE);
	printf("found OS object subtype %u lindex %u name %s\n",
	       (unsigned) os->attr->osdev.type, os->logical_index, os->name);
	assert(os->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS);
	if (strcmp(ibv_get_device_name(dev), os->name))
	  assert(0);
      }
    }
    hwloc_bitmap_free(set);
  }

  hwloc_topology_destroy(topology);

  ibv_free_device_list(dev_list);

  return 0;
}