int cuda_init_memopt(void) 
{
	int num_devices = cuda_devices();
	int device;
	int max_device = 0;

	if (num_devices > 1) {

		size_t mem_max = 0;
		size_t mem_free;
		size_t mem_total;

		for (device = 0; device < num_devices; device++) {

			cuda_init(device);
			CUDA_ERROR(cudaMemGetInfo(&mem_free, &mem_total));
			//printf(" device (%d): %d\n", device, mem_available);

			if (mem_max < mem_free) {

				mem_max = mem_free;
				max_device = device;
			}
		}
		//printf(" max device: %d\n", max_device);
		CUDA_ERROR(cudaSetDevice(max_device));
		// FIXME: we should set last_init
	}

	return max_device;
}

    /**
     * Returns information about device memory.
     *
     * @param free amount of free memory in bytes. can be NULL
     * @param total total amount of memory in bytes. can be NULL. (NULL by default)
     */
    void getMemoryInfo(size_t *free, size_t *total = NULL)
    {
        size_t freeInternal = 0;
        size_t totalInternal = 0;

        CUDA_CHECK(cudaMemGetInfo(&freeInternal, &totalInternal));

        if (free != NULL)
        {
            if (reservedMem > freeInternal)
                freeInternal = 0;
            else
                freeInternal -= reservedMem;

            *free = freeInternal;
        }
        if (total != NULL)
        {
            if (reservedMem > totalInternal)
                totalInternal = 0;
            else
                totalInternal -= reservedMem;

            *total = totalInternal;
        }
    }

cudaError_t THCudaMemGetInfoCached(THCState *state,  size_t* freeBytes, size_t* totalBytes, size_t* largestBlock)
{
  size_t cachedBytes = 0;
  THCDeviceAllocator* allocator = state->cudaDeviceAllocator;

  *largestBlock = 0;
  /* get info from CUDA first */
  cudaError_t ret = cudaMemGetInfo(freeBytes, totalBytes);
  if (ret!= cudaSuccess)
    return ret;

  int device;
  ret = cudaGetDevice(&device);
  if (ret!= cudaSuccess)
    return ret;

  /* not always true - our optimistic guess here */
  *largestBlock = *freeBytes;

  if (allocator->cacheInfo != NULL)
    allocator->cacheInfo(allocator->state, device, &cachedBytes, largestBlock);

  /* Adjust resulting free bytes number. largesBlock unused for now */
  *freeBytes += cachedBytes;
  return cudaSuccess;
}

// return free memory in megabytes
int cuda_available_memory(int thr_id)
{
	int dev_id = device_map[thr_id % MAX_GPUS];
	size_t mtotal, mfree = 0;
	cudaSetDevice(dev_id);
	cudaMemGetInfo(&mfree, &mtotal);
	return (int) (mfree / (1024 * 1024));
}

void getDeviceMemoryInfoInMb(int device, size_t *total, size_t *free) {
  static const int bytesInMb = 1024 * 1024;
  size_t freeInBytes;
  size_t totalInBytes;
  CHECK_ERR(cudaGetDevice(&device));
  CHECK_ERR(cudaMemGetInfo(&freeInBytes, &totalInBytes));
  *total = totalInBytes / bytesInMb;
  *free = freeInBytes / bytesInMb;
}

/** \brief Debug short device memory information (free/total) to stream if DEBUG
 *flag is set to true.
 *
 * @param force always print output
 * @param stream output stream
 */
inline void showMemoryInfo(bool force, FILE *stream)
{
  size_t free_mem = 0;
  size_t total_mem = 0;
  cudaMemGetInfo(&free_mem, &total_mem);
  if (DEBUG || force)
    fprintf(stream, "memory usage, free: %lu total: %lu\n", free_mem,
            total_mem);
}

static int cutorch_getMemoryUsage(lua_State *L) {
  size_t freeBytes = 0;
  size_t totalBytes = 0;
  int curDevice;
  THCudaCheck(cudaGetDevice(&curDevice));

  int device = luaL_optint(L, 1, -10);
  if (device == -10) { /* no argument passed, current device mem usage */
    THCudaCheck(cudaMemGetInfo(&freeBytes, &totalBytes));
  } else { /* argument was given, particular device's memory usage */
    THCudaCheck(cudaSetDevice(device-1)); /* zero indexed */
    THCudaCheck(cudaMemGetInfo(&freeBytes, &totalBytes));
    THCudaCheck(cudaSetDevice(curDevice));
  }
  lua_pushnumber(L, freeBytes);
  lua_pushnumber(L, totalBytes);
  return 2;
}

void oskar_device_mem_info(size_t* mem_free, size_t* mem_total)
{
    if (!mem_free || !mem_total) return;
#ifdef OSKAR_HAVE_CUDA
    cudaMemGetInfo(mem_free, mem_total);
#else
    (void) mem_free;
    (void) mem_total;
#endif
}

static KMCUDAResult print_memory_stats() {
  size_t free_bytes, total_bytes;
  if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) {
    return kmcudaRuntimeError;
  }
  printf("GPU memory: used %zu bytes (%.1f%%), free %zu bytes, total %zu bytes\n",
         total_bytes - free_bytes, (total_bytes - free_bytes) * 100.0 / total_bytes,
         free_bytes, total_bytes);
  return kmcudaSuccess;
}

inline void check_device_memory( const char* filename, const int line_number)
{
#ifdef CUDA_DEBUG
   size_t avail;
   size_t total;
   cudaMemGetInfo( &avail, &total);
   size_t used = total - avail;
   printf( "CUDA device memory usage at %s:%i: Used: %f Mb, Free %f Mb\n", filename, line_number, float(used)/(1024*1024), float(avail)/(1024*1024));
#endif
}

AllocPtr CudaDevice::CreateDefaultAlloc() {
	// Create the allocator. Use a bucket allocator with a capacity limit at
	// 80% of free mem.
	intrusive_ptr<CudaAllocBuckets> alloc(new CudaAllocBuckets(this));
	size_t freeMem, totalMem;

	cudaMemGetInfo(&freeMem, &totalMem);
	alloc->SetCapacity((size_t)(.80 * freeMem));
	
	return AllocPtr(alloc.get());
}

		std::ostream& operator<< (std::ostream& out, const cuda_running_configuration& running_configuration)
		{
			out << "--- CUDA versions ---" << std::endl;
			out << "Driver version = " << running_configuration.driver_version / 1000 << "." << (running_configuration.driver_version % 100) / 10 << std::endl;
			out << "Runtime version = " << running_configuration.runtime_version / 1000 << "." << (running_configuration.runtime_version % 100) / 10 << std::endl;

			out << "--- Device ---" << std::endl;

			out << "Device Id = " << running_configuration.device_id << std::endl;
			out << "Device name = " << running_configuration.device_name << std::endl;
			out << "Compute capability = " << running_configuration.compute_capability_major << "." << running_configuration.compute_capability_minor << std::endl;
			out << "Clock rate = " << (running_configuration.clock_rate / 1000) << " MHz" << std::endl;
			out << "Memory clock rate = " << (running_configuration.memory_clock_rate / 1000) << " MHz" << std::endl;
			out << "Memory bus width = " << running_configuration.memory_bus_width << " bits" << std::endl;
			out << "Global memory size = " << running_configuration.global_memory_size / (1024 * 1024) << " MB" << std::endl;
			out << "ECC support = " << (running_configuration.ecc_enabled ? "Enabled" : "Disabled") << std::endl;
			out << "L2 cache size = " << running_configuration.l2_cache_size << " bytes" << std::endl;
			out << "Multiprocessor count = " << running_configuration.multiprocessor_count << std::endl;
			out << "Shared memory per block size = " << running_configuration.smem_per_block << " bytes" << std::endl;
			out << "Maximum number of threads per multiprocessor = " << running_configuration.max_threads_per_multiprocessor << std::endl;
			out << "Maximum number of threads per block = " << running_configuration.max_threads_per_block << std::endl;
			out << "Maximum sizes of each dimension of a block = "
				<< running_configuration.max_threads_dim[0] << " x "
				<< running_configuration.max_threads_dim[1] << " x "
				<< running_configuration.max_threads_dim[2] << std::endl;
			out << "Maximum sizes of each dimension of a grid = "
				<< running_configuration.max_grid_size[0] << " x "
				<< running_configuration.max_grid_size[1] << " x "
				<< running_configuration.max_grid_size[2] << std::endl;
			out << "Maximum size of 1D texture bound to linear memory = " << running_configuration.max_texture_1d_linear << std::endl;
			out << "Texture alignment = " << running_configuration.texture_alignment << " bytes" << std::endl;
			out << "PCI Bus ID = " << running_configuration.pci_bus_id << std::endl;
			out << "PCI Location ID = " << running_configuration.pci_device_id << std::endl;
			#ifdef WIN32
				out << "Driver mode = " << (running_configuration.tcc_mode ? "TCC" : "WDDM") << std::endl;
			#endif

			out << "--- Settings ---" << std::endl;

			out << "Max global memory usage ratio = " << running_configuration.max_global_memory_usage_ratio << std::endl;

			out << "--- Status ---" << std::endl;

			size_t free_memory;
			size_t total_memory;
			cuda_safe_call(cudaMemGetInfo(&free_memory, &total_memory));

			out << "Free memory = " << free_memory / (1024 * 1024) << " MB" << std::endl;
			out << "Total memory = " << total_memory / (1024 * 1024) << " MB" << std::endl;

			return out;
		}

inline std::string
cudaMemoryInfoText()
{
    size_t free;
    size_t total;
    CUGIP_CHECK_RESULT(cudaMemGetInfo( &free, &total));

    return boost::str( boost::format("Free GPU memory: %1% MB; Total GPU memory %2% MB; Occupied %3%%%")
                       % (float(free) / (1024*1024))
                       % (float(total) / (1024*1024))
                       % (100.0f * float(total - free)/total)
                     );
}

int getMemUsage(const int &myRank)
{
    size_t free_byte,
           total_byte;

    CHECK(cudaMemGetInfo(&free_byte, &total_byte));

    std::cout << "myRank: " << myRank << " "
              << free_byte / 1024.0 / 1024.0 
              << " / " << total_byte / 1024.0 / 1024.0 << std::endl;

    return 0;
}

void memoryInfo(void)
{
	size_t free;
	size_t total;
	
	cudaCheck(cudaMemGetInfo (&free,&total),"MemInfo11");
	
	printf("\n");
	printf("\nRANK=%d\n",RANK);
	printf("\nGPU total memory = % .2f MB\n",(float)total/1e6);
	printf("\nGPU free  memory = % .2f MB\n",(float)free/1e6);

}

void AboutDialog::
        showEvent(QShowEvent *)
{
    ui->labelVersion->setText( QString::fromStdString( Sawe::Configuration::version_string() ) );
    ui->labelTimestamp->setText( QString("Built on %1 at %2 from revision %3.")
                                 .arg(Sawe::Configuration::build_date().c_str())
                                 .arg(Sawe::Configuration::build_time().c_str())
                                 .arg(Sawe::Configuration::revision().c_str()) );
    ui->labelLicense->setText( Sawe::Reader::reader_text().c_str() );
    if (Sawe::Reader::reader_title() == Sawe::Reader::reader_text() )
        ui->labelLicense->clear();

    int cores = Sawe::Configuration::cpuCores();

#ifdef USE_CUDA
    size_t free=0, total=0;
    cudaMemGetInfo(&free, &total);
    cudaDeviceProp prop = CudaProperties::getCudaDeviceProp();

    ui->labelSystem->setText(QString(
            "Using CPU with %13 core%14.\n"
            "Using GPU (%1 of %2) %3.\n"
            "%4 free of %5 total graphics memory.\n"
            "Gpu Gflops: %6\n"
            "Gpu memory speed: %7/s (estimated)\n"
            "Cpu memory speed: %8/s (estimated)\n"
            "Cuda compute capability: %9.%10\n"
            "Cuda driver version: %11\n"
            "Cuda runtime version: %12\n")
                             .arg(1+CudaProperties::getCudaCurrentDevice())
                             .arg(CudaProperties::getCudaDeviceCount())
                             .arg(prop.name)
                             .arg(DataStorageVoid::getMemorySizeText( free, 4 ).c_str())
                             .arg(DataStorageVoid::getMemorySizeText( total, 4 ).c_str())
                             .arg(CudaProperties::flops(prop)*1e-9, 0, 'f', 0)
                             .arg(DataStorageVoid::getMemorySizeText( CudaProperties::gpu_memory_speed(), 1, 'f' ).c_str())
                             .arg(DataStorageVoid::getMemorySizeText( CpuProperties::cpu_memory_speed(), 1, 'f' ).c_str())
                             .arg(prop.major).arg(prop.minor)
                             .arg(CudaProperties::getCudaDriverVersion())
                             .arg(CudaProperties::getCudaRuntimeVersion())
                             .arg(cores).arg(cores==1?"":"s")
                             );
#else
    ui->labelSystem->setText(QString(
            "Using CPU with %2 core%3.\n"
            "Cpu memory speed: %1 GB/s (estimated)\n")
                             .arg(CpuProperties::cpu_memory_speed()*1e-9, 0, 'f', 1)
                             .arg(cores).arg(cores==1?"":"s")
                             );
#endif
}

__declspec(dllexport) long __stdcall GetTotalMemory(int device)
{
    int currentdevice = 0;
    cudaGetDevice(&currentdevice);

    cudaSetDevice(device);

    size_t freemem = 0, totalmem = 0;
    cudaMemGetInfo(&freemem, &totalmem);

    cudaSetDevice(currentdevice);

    return (long)(totalmem >> 20);
}

/**
 * \brief Creates and initializes the working data for the plan
 * \param [in] plan The data and memory location for the plan.
 * \return int Error flag value
 * \sa parseCUDAMEMPlan 
 * \sa makeCUDAMEMPlan
 * \sa execCUDAMEMPlan
 * \sa perfCUDAMEMPlan
 * \sa killCUDAMEMPlan
*/
int   initCUDAMEMPlan(void *plan) {
	size_t avail, total, arraybytes;
	int M,i;
	int ret = make_error(ALLOC,generic_err);
	double gputhreads;
	cudaError_t cudaStat;
	struct cudaDeviceProp prop;
	Plan *p;
	CUDAMEMdata *d = NULL;
	p = (Plan *)plan;
	if (p) {
		d = (CUDAMEMdata*)p->vptr;
		p->exec_count = 0;
		perftimer_init(&p->timers, NUM_TIMERS);
	}
	if(d) {
		CUDA_CALL( cudaSetDevice(d->device) );
		CUDA_CALL( cudaMemGetInfo(&avail, &total) );
		CUDA_CALL( cudaGetDeviceProperties(&prop, d->device) );
		if (d->nGpuThreads != 0) {	// use the user spec'd number of threads or default to warp*cores
			gputhreads = (double)(d->nGpuThreads);
		} else {
			gputhreads = d->nGpuThreads = prop.warpSize * prop.multiProcessorCount;
		}
		if (prop.major < 2) {	// check results on older devices
			d->resultCheck = 1;
		} else {
			d->resultCheck = 0;
		}
		// calculate M for 6 M*M arrays to fill 100%/75%/50% of GPU free memory 
		// M = (d->nGpuThreads) * (int)(sqrt(0.75*avail/(6.0*sizeof(double)*gputhreads*gputhreads)));
		// M = (d->nGpuThreads) * (int)(sqrt(0.50*avail/(6.0*sizeof(double)*gputhreads*gputhreads)));
		M = (d->nGpuThreads) * (int)(sqrt(1.00*avail/(6.0*sizeof(double)*gputhreads*gputhreads)));
		// assume one will fit in host memory
		d->M = M;
		arraybytes = (size_t)(0.99*avail);
		d->arraybytes = arraybytes;
                d->arrayelems = arraybytes / sizeof(int);
		// host array and device arrays
		CUDA_CALL( cudaMallocHost((void **)(&(d->hostarray)), arraybytes) );
		CUDA_CALL( cudaMalloc    ((void **)(&(d->devicearray)), arraybytes) );
		// initialize so that results are M*PI**2/100
		//for(i=0; i<3*M*M; i++) d->HA[i] = (double)0.31415926535;
		//CUDA_CALL( cudaMemcpy( (d->DA), (d->HA), arraybytes, cudaMemcpyHostToDevice) );
		//CUDA_CALL( cudaMemcpy( (d->DB), (d->DA), arraybytes, cudaMemcpyDeviceToDevice) );
		ret = ERR_CLEAN;
	}
	return ret;
}

static int cutorch_getDeviceProperties(lua_State *L)
{
  int device = (int)luaL_checknumber(L, 1)-1;

  // switch context to given device so the call to cudaMemGetInfo is for the correct device
  int oldDevice;
  THCudaCheck(cudaGetDevice(&oldDevice));
  THCudaCheck(cudaSetDevice(device));

  struct cudaDeviceProp prop;
  THCudaCheck(cudaGetDeviceProperties(&prop, device));
  lua_newtable(L);
  SET_DEVN_PROP(canMapHostMemory);
  SET_DEVN_PROP(clockRate);
  SET_DEVN_PROP(computeMode);
  SET_DEVN_PROP(deviceOverlap);
  SET_DEVN_PROP(integrated);
  SET_DEVN_PROP(kernelExecTimeoutEnabled);
  SET_DEVN_PROP(major);
  SET_DEVN_PROP(maxThreadsPerBlock);
  SET_DEVN_PROP(memPitch);
  SET_DEVN_PROP(minor);
  SET_DEVN_PROP(multiProcessorCount);
  SET_DEVN_PROP(regsPerBlock);
  SET_DEVN_PROP(sharedMemPerBlock);
  SET_DEVN_PROP(textureAlignment);
  SET_DEVN_PROP(totalConstMem);
  SET_DEVN_PROP(totalGlobalMem);
  SET_DEVN_PROP(warpSize);
  SET_DEVN_PROP(pciBusID);
  SET_DEVN_PROP(pciDeviceID);
  SET_DEVN_PROP(pciDomainID);
  SET_DEVN_PROP(maxTexture1D);
  SET_DEVN_PROP(maxTexture1DLinear);

  size_t freeMem;
  THCudaCheck(cudaMemGetInfo (&freeMem, NULL));
  lua_pushnumber(L, freeMem);
  lua_setfield(L, -2, "freeGlobalMem");

  lua_pushstring(L, prop.name);
  lua_setfield(L, -2, "name");

  // restore context
  THCudaCheck(cudaSetDevice(oldDevice));

  return 1;
}

int main(int argc, char **argv)
{
  cudaError_t err = cudaSuccess;
  int deviceCount = 0;
  size_t totalDevMem, freeDevMem;
  size_t lastLineLength = 0; // MUST be initialized to zero

  signal(SIGTERM, signalHandler);
  signal(SIGQUIT, signalHandler);
  signal(SIGINT, signalHandler);
  signal(SIGHUP, signalHandler);

  writeLine(lastLineLength, "Preparing...");

  err = cudaGetDeviceCount(&deviceCount);

  if (err != cudaSuccess) {
   std::cerr << "ERROR: " << cudaGetErrorString(err) << std::endl; 
  }

  while (err == cudaSuccess && gRun) {
    
    std::ostringstream stream;

    for (int i=0; i < deviceCount; ++i) {
      if (err == cudaSuccess) {
	err = cudaSetDevice(i);
	if (err == cudaSuccess) {
	  cudaMemGetInfo(&freeDevMem, &totalDevMem);
	  if (i != 0)
	    stream << " : ";
	  stream << "Dev " << i << " (" << (freeDevMem/1024) << " KB of " << (totalDevMem/1048576) << " MB free)";
	}
      }
    }
    if (err == cudaSuccess) {
      writeLine(lastLineLength, stream.str());
    }
    
    sleep(5); // TODO - make the cycle time an optional command line flag...
  }

  cudaThreadExit();

  std::cout << std::endl;

  return 0;
}