int BulkAdapt::get_node_from_idxl(int node_idxl, int partID)
{
  IDXL_List *list = meshPtr->node.shared.getIdxlListN(partID);
  CkAssert(list!=NULL);
  CkAssert(list->size()>node_idxl);
  return (*list)[node_idxl];
}

/**  Print all the elements adjacent to the given element.
*/
void FEM_Mesh::e2e_printAll(ElemID e) 
{
	if (e.id == -1) return; // non existent element
	
	FEM_IndexAttribute *eAdj;
	FEM_IndexAttribute *eAdjType;
	
	if(FEM_Is_ghost_index(e.id)){
		eAdj = (FEM_IndexAttribute *)elem[e.type].getGhost()->lookup(FEM_ELEM_ELEM_ADJACENCY,"e2e_printAll");
		eAdjType = (FEM_IndexAttribute *)elem[e.type].getGhost()->lookup(FEM_ELEM_ELEM_ADJ_TYPES,"e2e_printAll");
	}
	else {
		eAdj = (FEM_IndexAttribute *)elem[e.type].lookup(FEM_ELEM_ELEM_ADJACENCY,"e2e_printAll");
		eAdjType = (FEM_IndexAttribute *)elem[e.type].lookup(FEM_ELEM_ELEM_ADJ_TYPES,"e2e_printAll");
	}
	
	
	AllocTable2d<int> &eAdjs = eAdj->get();
	AllocTable2d<int> &eAdjTypes = eAdjType->get();
	CkAssert(eAdjs.width() == eAdjTypes.width());

	CkAssert(e.getSignedId()>=0);

	for (int i=0; i<eAdjs.width(); i++) {
		CkPrintf("Element %d,%d is adjacent to %d,%d\n", e.type, e.id, eAdjTypes[e.getSignedId()][i], eAdjs[e.getSignedId()][i]);

	}

}

/** Count the number of elements on edge (n1, n2) */
int FEM_Mesh::countElementsOnEdge(int n1, int n2) {
  if (n1==n2) {
    CkPrintf("ERROR: You called countElementsOnEdge() with two identical nodes %d, and %d \n", n1, n2);
    CkExit();
  }
  
  int *n1AdjElems=0, *n2AdjElems=0;
  int n1NumElems, n2NumElems;

  CkAssert(node.is_valid_any_idx(n1));
  CkAssert(node.is_valid_any_idx(n2));

  n2e_getAll(n1, n1AdjElems, n1NumElems);
  n2e_getAll(n2, n2AdjElems, n2NumElems);
  CkAssert(n1AdjElems!=0);
  CkAssert(n2AdjElems!=0);
  int count=0;


  for (int i=0; i<n1NumElems; i++) {
    for (int j=0; j<n2NumElems; j++) {
      if (n1AdjElems[i] == n2AdjElems[j]) {
        count++;
      }
    }
  }
  delete[] n1AdjElems;
  delete[] n2AdjElems;

  return count;
}

//! one entry is called for 'time' seconds, value is counter reading
void StatTable::setEp(int epidx, int stat, long long value, double time) 
{
  // CmiPrintf("StatTable::setEp %08x %d %d %d %f\n", 
  //           this, epidx, stat, value, time);

  CkAssert(epidx<MAX_ENTRIES);
  CkAssert(stat<numStats_ && stat>=0);
  
  int numCalled = stats_[stat].numCalled[epidx];
  double avg = stats_[stat].avgCount[epidx];
  double stdDev = stats_[stat].stdDevCount[epidx];
  stats_[stat].numCalled[epidx]++;
  stats_[stat].avgCount[epidx] = (avg * numCalled + value) / (numCalled + 1);
  // note that the stddev calculation is not quite correct, but it's
  // almost correct and over time will converge with true value
  avg = stats_[stat].avgCount[epidx];
  stats_[stat].stdDevCount[epidx] = 
    (stdDev * numCalled + (value-avg)*(value-avg)) / (numCalled+1);
  // CmiPrintf("variance %f avg %f value %ld calcVariance %f\n", 
  // stdDev, avg, value, stats_[stat].stdDevCount[epidx]);
  stats_[stat].totTime[epidx] += time;
  if (stats_[stat].maxCount[epidx] < value) {
    stats_[stat].maxCount[epidx] = value;
  }
  if (stats_[stat].minCount[epidx] > value) {
    stats_[stat].minCount[epidx] = value;
  }
}

// non-recursive algorithm with adaptive grain size control
void Node::sequentialColoringHelper(bool& wait, bool& solutionFound, std::vector<vertex>& result)
{
  while(!stackForSequential.empty())
  {
    sequentialCurr = CkTimer();

    // Check if the sequential algorithm has run for 'timeout' seconds. If yes,
    // then return and call rerun() entry method. That would execute the
    // remaining stack.
    if(sequentialCurr - sequentialStart > timeout)
    {
#ifdef DEBUG
      CkPrintf("Timeout in sequential coloring. Stack size=%d\n", stackForSequential.size());
#endif
      thisProxy.rerun();
      wait = true;
      return;
    }

    stackNode curr_node_ = stackForSequential.top().first;
    std::stack<int> removedVertices = stackForSequential.top().second;
    stackForSequential.pop();  // remove from stack

    // vertex removal step
    curr_node_.uncolored_num_ -= curr_node_.vertexRemoval(removedVertices);

    // coloring found
    if(curr_node_.uncolored_num_==0)
    {
      solutionFound = true;
      curr_node_.mergeRemovedVerticesBack(removedVertices);
      result = curr_node_.node_state_;
      return;
    }

    int vIndex = curr_node_.getNextConstrainedVertex();
    CkAssert(vIndex!=-1);

    // Value Ordering
    // temporary stack to invert the priority queue ordering
    std::stack<stackNode> tmp;
    pq_type priorityColors = curr_node_.getValueOrderingOfColors(vIndex);
    while(!priorityColors.empty()){
      std::pair<int,int> p = priorityColors.top();
      priorityColors.pop();
      std::vector<vertex> new_state = curr_node_.node_state_;
      CkAssert(p.first < chromaticNum_);
      int verticesColored = curr_node_.updateState(new_state, vIndex, p.first, true);
      CkAssert(verticesColored >= 1);
      tmp.emplace(stackNode(new_state, curr_node_.uncolored_num_ - verticesColored));
    }

    while(!tmp.empty())
    {
      stackForSequential.push(std::make_pair(tmp.top(), removedVertices));
      tmp.pop();
    }
  }

}

/*  Coloring clique (i.j,k).
 *  assign min possible color to i and update its ngh.
 *  Do the same for j and k.
 *  While we are doing clique coloring we are also doing 
 *  forced moves (using updateState) which will give
 *  further improvements.
 */
void Node::colorClique3(int i, int j, int k)
{
  if(false == node_state_[i].isColored()) {
    boost::dynamic_bitset<> i_possC = node_state_[i].getPossibleColor();
    size_t i_color = i_possC.find_first();
    CkAssert(i_color != boost::dynamic_bitset<>::npos);
    int verticesColored =  updateState(node_state_, i, i_color , true);
    uncolored_num_  -=  verticesColored;
  }

  if(false == node_state_[j].isColored()) {
    boost::dynamic_bitset<> j_possC = node_state_[j].getPossibleColor();
    size_t j_color = j_possC.find_first();
    CkAssert(j_color != boost::dynamic_bitset<>::npos);
    int verticesColored = updateState(node_state_, j, j_color , true);
    uncolored_num_ -= verticesColored;
  }

  if(false == node_state_[k].isColored()) {
    boost::dynamic_bitset<> k_possC = node_state_[k].getPossibleColor();
    size_t k_color = k_possC.find_first();
    CkAssert(k_color != boost::dynamic_bitset<>::npos);
    int verticesColored = updateState(node_state_, k, k_color, true );
    uncolored_num_ -= verticesColored;
  }
}

/** Adds newElem to node n's element adjacency list
 */
void FEM_Mesh::n2e_add(int n, int newElem) 
{
  if (n == -1) return;
  if(FEM_Is_ghost_index(n)){
	FEM_VarIndexAttribute *eAdj = (FEM_VarIndexAttribute *)node.getGhost()->lookup(FEM_NODE_ELEM_ADJACENCY,"n2e_add");     
	CkVec<CkVec<ElemID> > &eVec = eAdj->get();
	CkVec<ElemID> &nsVec = eVec[FEM_To_ghost_index(n)];
	ElemID ne(0, newElem);
	nsVec.push_back(ne);
	int *testn2e, testn2ec;
	n2e_getAll(n,testn2e,testn2ec);
	for(int i=0; i<testn2ec; i++) {
	  if(FEM_Is_ghost_index(testn2e[i]))
	    CkAssert(elem[0].ghost->is_valid(FEM_From_ghost_index(testn2e[i])));
	  else 
	    CkAssert(elem[0].is_valid(testn2e[i]));
	}
	if(testn2ec!=0) delete[] testn2e;
  }
  else {
	FEM_VarIndexAttribute *eAdj = (FEM_VarIndexAttribute *)node.lookup(FEM_NODE_ELEM_ADJACENCY,"n2e_add");     
	CkVec<CkVec<ElemID> > &eVec = eAdj->get();
	CkVec<ElemID> &nsVec = eVec[n];
	ElemID ne(0, newElem);
	nsVec.push_back(ne);
  }
}

/// Update send/recv table at timestamp
void PVT::objUpdate(POSE_TimeType timestamp, int sr)
{
#ifndef CMK_OPTIMIZE
  int tstat = localStats->TimerRunning();
  if(pose_config.stats){
    if (tstat)
      localStats->SwitchTimer(GVT_TIMER);
    else
      localStats->TimerStart(GVT_TIMER);
  }
#endif
  //if ((timestamp < estGVT) && (estGVT > POSE_UnsetTS))
  //CkPrintf("timestamp=%d estGVT=%d simdone=%d sr=%d\n", timestamp,
  //estGVT, simdone, sr);
  CkAssert(simdone>0 || (timestamp >= estGVT) || (estGVT == POSE_UnsetTS));
  CkAssert((sr == SEND) || (sr == RECV));
  if ((estGVT > POSE_UnsetTS) && 
      ((timestamp < iterMin) || (iterMin == POSE_UnsetTS))) 
    iterMin = timestamp;
  if (waitForFirst) {
    waitForFirst = 0;
    SendsAndRecvs->Restructure(estGVT, timestamp, sr);
  }
  else SendsAndRecvs->Insert(timestamp, sr);
#ifndef CMK_OPTIMIZE
  if(pose_config.stats){
    if (tstat)
      localStats->SwitchTimer(tstat);
    else
      localStats->TimerStop();
  }
#endif

}

/** Finds oldElem in n's element adjacency list, and replaces it with newElem
 */
void FEM_Mesh::n2e_replace(int n, int oldElem, int newElem) 
{
  if (n == -1) return;
  if(FEM_Is_ghost_index(n)){
	FEM_VarIndexAttribute *eAdj = (FEM_VarIndexAttribute *)node.getGhost()->lookup(FEM_NODE_ELEM_ADJACENCY,"n2e_replace");
	CkVec<CkVec<ElemID> > &eVec = eAdj->get();
	CkVec<ElemID> &nsVec = eVec[FEM_To_ghost_index(n)];
	for (int i=0; i<nsVec.length(); i++) {
	  if (nsVec[i].getSignedId() == oldElem) {
		nsVec[i] = ElemID(0,newElem);
		break;
	  }
	}
	int *testn2e, testn2ec;
	n2e_getAll(n,testn2e,testn2ec);
	for(int i=0; i<testn2ec; i++) {
	  if(FEM_Is_ghost_index(testn2e[i]))
	    CkAssert(elem[0].ghost->is_valid(FEM_From_ghost_index(testn2e[i])));
	  else 
	    CkAssert(elem[0].is_valid(testn2e[i]));
	}
	if(testn2ec!=0) delete[] testn2e;
  }
  else{
	FEM_VarIndexAttribute *eAdj = (FEM_VarIndexAttribute *)node.lookup(FEM_NODE_ELEM_ADJACENCY,"n2e_replace");
	CkVec<CkVec<ElemID> > &eVec = eAdj->get();
	CkVec<ElemID> &nsVec = eVec[n];
	for (int i=0; i<nsVec.length(); i++) {
	  if (nsVec[i].getSignedId() == oldElem) {
		nsVec[i] = ElemID(0,newElem);
		break;
	  }
	}
  }
}

/// In sequential mode, begin checkpoint after reaching quiescence
void sim::SeqBeginCheckpoint() {
  // Ensure this only happens on sim 0
  CkAssert(thisIndex == 0);
  // Ensure we're checkpointing
  CkAssert(seqCheckpointInProgress);
  CkPrintf("POSE: quiescence detected\n");
  CkPrintf("POSE: beginning checkpoint on sim %d at GVT=%lld sim time=%.1f sec\n", thisIndex, seqLastCheckpointGVT, CmiWallTimer() + seqStartTime);
  CkCallback cb(CkIndex_sim::SeqResumeAfterCheckpoint(), CkArrayIndex1D(thisIndex), thisProxy);
  CkStartCheckpoint(POSE_CHECKPOINT_DIRECTORY, cb);
}

const eventID& GetEventID() {
  //CpvStaticDeclare(eventID, theEventID);  // initializes to [0.pe]
  //  for each pe called on
  CpvAccess(theEventID).incEventID();
  CkAssert(CpvAccess(theEventID).getPE()>=0);
  return(CpvAccess(theEventID));
 }

/// Start a forward execution step on myStrat after a checkpoint (sequential mode only)
void sim::CheckpointStep(eventMsg *m) {
  CkFreeMsg(m);
  if (active < 0) return; // object is migrating; deactivate it 
#if !CMK_TRACE_DISABLED
  int tstat;
  if (pose_config.stats) {
    tstat = localStats->TimerRunning();
    if (!tstat)
      localStats->TimerStart(SIM_TIMER);
    else localStats->SwitchTimer(SIM_TIMER);
  }
#endif

  // ensure sequential mode
  CkAssert(myStrat->STRAT_T == SEQ_T);
  myStrat->Step(); // Call Step on strategy

#if !CMK_TRACE_DISABLED
  if (pose_config.stats) {
    if (!tstat)
      localStats->TimerStop();
    else localStats->SwitchTimer(tstat);
  }
#endif
}

/* -----------------------------------------
 * This is the node state BEFORE the search is initiated.
 * 1. initialize node_state_ with adjList all uncolored
 * 2. Does pre-coloring
 * ---------------------------------------*/
Node::Node(bool isRoot, int n, CProxy_Node parent) :
  nodeID_("0"), parent_(parent), uncolored_num_(n), is_root_(isRoot),
  child_num_(0), child_finished_(0),
  child_succeed_(0), is_and_node_(false), parentBits(1), parentPtr(NULL)

{
  __sdag_init();
  programStart = CkTimer();
  CkAssert(isRoot);  
  vertex v = vertex(chromaticNum_);
  node_state_ = std::vector<vertex>(vertices_, v);

  //preColor();
#ifdef  DEBUG 
  CkPrintf("After Precolor\n");
  printGraph();
#endif 

  int count = uncolored_num_;
  uncolored_num_ -= vertexRemoval();
  CkPrintf("Vertices removed by vertex removal in [MainChare] = %d\n", count-uncolored_num_);

#ifdef  DEBUG 
  CkPrintf("Vertex Removal\n");
  printGraph();
#endif 

  CProxy_counter(counterGroup).ckLocalBranch()->registerMe(nodeID_);
  run();
}

  void SayHi(HiMsg *m)
  {
    redNo ++;
    CmiAssert(m->data[0] == 22 && m->data[1] == 28);

    CkGetSectionInfo(sid, m);

    CkMulticastMgr *mg = CProxy_CkMulticastMgr(mCastGrpId).ckLocalBranch();
    int dataSize = (int)(CompleteMsgSize);
    int* data = new int [dataSize];
    int fragSize = dataSize/NumFrags;
    CkAssert (0 != fragSize);
    for (int i=0; i<dataSize; i++) {
      data [i] = thisIndex;
    }
    CkCallback cb(CkIndex_Hello::cb_client(NULL), CkArrayIndex1D(0), thisProxy);
    mg->contribute(dataSize*sizeof(int), data,CkReduction::sum_int, sid, cb, fragSize*sizeof(int));
//    data[0] = thisIndex+2;
//    data[1] = thisIndex+2;
//    mg->contribute(2*sizeof(int), &data,CkReduction::max_int, sid, sizeof(int));
//    data[0] = thisIndex+1;
//    data[1] = thisIndex+1;
//    mg->contribute(2*sizeof(int), &data,CkReduction::product_int, sid, sizeof(int));
    delete m;
    if (1)
      ckMigrate((CkMyPe()+1)%CkNumPes());
  }

void * rectRequest (int comm) {
  //  fprintf(stderr,"[%d] rectRequest for %d gives %p\n",CkMyPe(),comm,CkpvAccess(com_rect_ptr)->get(comm));
#ifdef COMLIB_RECT_DEBUG
  CkAssert(CkpvAccess(com_rect_ptr)->get(comm)!=NULL);
  isSane(CkpvAccess(com_rect_ptr)->get(comm),comm);
#endif
  return CkpvAccess(com_rect_ptr)->get(comm);
}

void _libExitHandler(envelope *env)
{
	switch(env->getMsgtype()) {
		case StartExitMsg:
			CkAssert(CkMyPe()==0);
			// else goto next
		case ExitMsg:
			CkAssert(CkMyPe()==0);
			if(_libExitStarted) {
				CmiFree(env);
				return;
			}
			_libExitStarted = 1;
			env->setMsgtype(ReqStatMsg);
			env->setSrcPe(CkMyPe());
			// if exit in ring, instead of broadcasting, send in ring
			if (_ringexit){
				const int stride = CkNumPes()/_ringtoken;
				int pe = 0; while (pe<CkNumPes()) {
					CmiSyncSend(pe, env->getTotalsize(), (char *)env);
					pe += stride;
				}
				CmiFree(env);
			}else{
				CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
			}	
			break;
		case ReqStatMsg:
			if (_ringexit) {
				int stride = CkNumPes()/_ringtoken;
				int pe = CkMyPe()+1;
				if (pe < CkNumPes() && pe % stride != 0)
					CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
				else
					CmiFree(env);
			}
			else
				CmiFree(env);
			//everyone exits here - there may be issues with leftover messages in the queue
			_libExitStarted = 0;
			CpvAccess(charmLibExitFlag) = 1;
			break;
		default:
			CmiAbort("Internal Error(_libExitHandler): Unknown-msg-type. Contact Developers.\n");
	}
}

void FEM_mesh_smooth(int mesh, int *nodes, int nNodes, int attrNo)
{
  vector2d newPos, *coords, *ghostCoords,theCoord;
  int nNod, nGn, *boundVals, nodesInChunk;
  int neighbors[3], *adjnodes;
  int gIdxN;
  FEM_Mesh *meshP = FEM_Mesh_lookup(mesh, "driver");
  nodesInChunk = FEM_Mesh_get_length(mesh,FEM_NODE);
  boundVals = new int[nodesInChunk];
  nGn = FEM_Mesh_get_length(mesh, FEM_GHOST + FEM_NODE);
  coords = new vector2d[nodesInChunk+nGn];

  FEM_Mesh_data(mesh, FEM_NODE, FEM_BOUNDARY, (int*) boundVals, 0, nodesInChunk, FEM_INT, 1);    

  FEM_Mesh_data(mesh, FEM_NODE, attrNo, (double*)coords, 0, nodesInChunk, FEM_DOUBLE, 2);

  IDXL_Layout_t coord_layout = IDXL_Layout_create(IDXL_DOUBLE, 2);
  FEM_Update_ghost_field(coord_layout,-1, coords); 
  ghostCoords = &(coords[nodesInChunk]);
 // FEM_Mesh_data(FEM_Mesh_default_write(), FEM_GHOST+FEM_NODE, attrNo, (double*)ghostCoords, 0, nGn, FEM_DOUBLE, 2);
  for (int i=0; i<nNodes; i++)
  {
    newPos.x=0;
    newPos.y=0;
    CkAssert(nodes[i]<nodesInChunk);  
    if (FEM_is_valid(mesh, FEM_NODE, i) && boundVals[i]>-1) //node must be internal
    {
      meshP->n2n_getAll(i, &adjnodes, &nNod);
     // for (int j=0; j<nNod; j++) CkPrintf("node[%d]: %d\n", i,adjnodes[j]);
      
      for (int j=0; j<nNod; j++) { //for all adjacent nodes, find coords
	if (adjnodes[j]<-1) {
	  gIdxN = FEM_From_ghost_index(adjnodes[j]);
	  newPos.x += theCoord.x=ghostCoords[gIdxN].x;
	  newPos.y += theCoord.y=ghostCoords[gIdxN].y;
	}
	else {
	  newPos.x += theCoord.x=coords[adjnodes[j]].x;
	  newPos.y += theCoord.y=coords[adjnodes[j]].y;
	}     
	int rank=FEM_My_partition();
	if (rank==0 && i==3 || rank==1 && i==17) {
	  CkPrintf("node[%d]: %d\n", i,adjnodes[j]);
	  CkPrintf("%d: (%f, %f)\n", j, theCoord.x, theCoord.y);
	}
      }
      newPos.x/=nNod;
      newPos.y/=nNod;
      FEM_set_entity_coord2(mesh, FEM_NODE, nodes[i], newPos.x, newPos.y);
      delete [] adjnodes;
    }
  }
 // FEM_Update_field(coord_layout, coords);
 // FEM_Mesh_data(FEM_Mesh_default_write(), FEM_NODE, attrNo, (double*)coords, 0, nodesInChunk, FEM_DOUBLE, 2);

  if (coords) delete [] coords;
  delete [] boundVals;
}

main::main (CkArgMsg *m) {
  int sizeX=0, sizeY=0, sizeZ=0;
  int grainX=0, grainY=0, grainZ=0;

  if (m->argc <= 1) {
    sizeX = sizeY = sizeZ = 16;
    grainX = grainY = grainZ = 4;
  }
  else if (m->argc == 7) {
    sizeX = atoi (m->argv[1]);
    sizeY = atoi (m->argv[2]);
    sizeZ = atoi (m->argv[3]);

    grainX = atoi (m->argv[4]);
    grainY = atoi (m->argv[5]);
    grainZ = atoi (m->argv[6]);

    CkAssert ((sizeX % grainX) == 0);
    CkAssert ((sizeY % grainY) == 0);
    CkAssert ((sizeZ % grainZ) == 0);
  }
  else {
    sizeX = sizeY = sizeZ = atoi (m->argv[1]);
    grainX = grainY = grainZ = atoi (m->argv[2]);    
    CkAssert ((sizeX % grainX) == 0);
  }

  CkPrintf ("Calling Configure\n");
  configureLineFFTInfo (&info, sizeX, sizeY, sizeZ, 
			grainX, grainY, grainZ,
			NULL, 
			ARRAY_REDUCTION, 
			true,
			NUM_FFT_ITER);

  CkPrintf ("Calling Create\n");
  createLineFFTArray (&info);

  info.xProxy.setReductionClient (red_handler, NULL);

  CkPrintf ("Calling Start\n");
  startLineFFTArray (&info);

  delete m;
};

varsizetest2_main::varsizetest2_main(void)
{
  int sizes[2];
  sizes[0] = 100; sizes[1] = 300;
  varsizetest2_Msg *m = new (sizes,0) varsizetest2_Msg();
  CkAssert(m->iarray && m->farray);
  m->myMain = thishandle;
  CProxy_varsizetest2_test::ckNew(m);
}

void CentralLB::findSimResults(LDStats* stats, int count, LBMigrateMsg* msg, LBSimulation* simResults)
{
    CkAssert(simResults != NULL && count == simResults->numPes);
    // estimate the new loads of the processors. As a first approximation, this is the
    // sum of the cpu times of the objects on that processor
    double startT = CkWallTimer();
    getPredictedLoadWithMsg(stats, count, msg, simResults->lbinfo, 1);
    CmiPrintf("getPredictedLoad finished in %fs\n", CkWallTimer()-startT);
}