int main(int argc, char *argv[])
{
    int errs = 0;
    MPI_Datatype outtype, oldtypes[2];
    MPI_Aint offsets[2];
    int blklens[2];
    MPI_Comm comm;
    int size, rank, src, dest, tag;

    MTest_Init(&argc, &argv);

    comm = MPI_COMM_WORLD;

    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &size);

    if (size < 2) {
        errs++;
        printf("This test requires at least 2 processes\n");
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

    src = 0;
    dest = 1;

    if (rank == src) {
        int buf[128], position, cnt;
        MTEST_VG_MEM_INIT(buf, 128 * sizeof(buf[0]));
        /* sender */

        /* Create a datatype and send it (multiple of sizeof(int)) */
        /* Create a send struct type */
        oldtypes[0] = MPI_INT;
        oldtypes[1] = MPI_CHAR;
        blklens[0] = 1;
        blklens[1] = 4 * sizeof(int);
        offsets[0] = 0;
        offsets[1] = sizeof(int);
        MPI_Type_struct(2, blklens, offsets, oldtypes, &outtype);
        MPI_Type_commit(&outtype);

        buf[0] = 4 * sizeof(int);
        /* printf("About to send to %d\n", dest); */
        MPI_Send(buf, 1, outtype, dest, 0, comm);
        MPI_Type_free(&outtype);

        /* Create a datatype and send it (not a multiple of sizeof(int)) */
        /* Create a send struct type */
        oldtypes[0] = MPI_INT;
        oldtypes[1] = MPI_CHAR;
        blklens[0] = 1;
        blklens[1] = 4 * sizeof(int) + 1;
        offsets[0] = 0;
        offsets[1] = sizeof(int);
        MPI_Type_struct(2, blklens, offsets, oldtypes, &outtype);
        MPI_Type_commit(&outtype);

        buf[0] = 4 * sizeof(int) + 1;
        MPI_Send(buf, 1, outtype, dest, 1, comm);
        MPI_Type_free(&outtype);

        /* Pack data and send as packed */
        position = 0;
        cnt = 7;
        MPI_Pack(&cnt, 1, MPI_INT, buf, 128 * sizeof(int), &position, comm);
        MPI_Pack((void *) "message", 7, MPI_CHAR, buf, 128 * sizeof(int), &position, comm);
        MPI_Send(buf, position, MPI_PACKED, dest, 2, comm);
    }
    else if (rank == dest) {
        MPI_Status status;
        int buf[128], i, elms, count;

        /* Receiver */
        /* Create a receive struct type */
        oldtypes[0] = MPI_INT;
        oldtypes[1] = MPI_CHAR;
        blklens[0] = 1;
        blklens[1] = 256;
        offsets[0] = 0;
        offsets[1] = sizeof(int);
        MPI_Type_struct(2, blklens, offsets, oldtypes, &outtype);
        MPI_Type_commit(&outtype);

        for (i = 0; i < 3; i++) {
            tag = i;
            /* printf("about to receive tag %d from %d\n", i, src); */
            MPI_Recv(buf, 1, outtype, src, tag, comm, &status);
            MPI_Get_elements(&status, outtype, &elms);
            if (elms != buf[0] + 1) {
                errs++;
                printf("For test %d, Get elements gave %d but should be %d\n", i, elms, buf[0] + 1);
            }
            MPI_Get_count(&status, outtype, &count);
            if (count != MPI_UNDEFINED) {
                errs++;
                printf("For partial send, Get_count did not return MPI_UNDEFINED\n");
            }
        }
        MPI_Type_free(&outtype);
    }
//.........这里部分代码省略.........

//.........这里部分代码省略.........
        MPI_Win_create(counter_mem, pof2*2*sizeof(int), sizeof(int),
                       MPI_INFO_NULL, MPI_COMM_WORLD, &win);

        MPI_Win_free(&win); 
        free(counter_mem);

        /* gather the results from other processes, sort them, and check 
           whether they represent a counter being incremented by 1 */

        results = (int *) malloc(NTIMES*nprocs*sizeof(int));
        for (i=0; i<NTIMES*nprocs; i++)
            results[i] = -1;

        MPI_Gather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, results, NTIMES, MPI_INT, 
                   0, MPI_COMM_WORLD);

        qsort(results+NTIMES, NTIMES*(nprocs-1), sizeof(int), compar);

        for (i=NTIMES+1; i<(NTIMES*nprocs); i++)
            if (results[i] != results[i-1] + 1)
                errs++;
        
        free(results);
    }
    else {
        /* Get the largest power of two smaller than nprocs */ 
        mask = 1; 
        nlevels = 0;
        while (mask < nprocs) {
            mask <<= 1; 
            nlevels++;
        }
        mask >>= 1;

        get_array = (int *) malloc(nlevels * sizeof(int));
        get_idx = (int *) malloc(nlevels * sizeof(int));
        acc_idx = (int *) malloc(nlevels * sizeof(int));

        level = 0; 
        idx   = 0; 
        tmp_rank = rank;
        while (mask >= 1) { 
            if (tmp_rank < mask) { 
                /* go to left for acc_idx, go to right for
                   get_idx. set idx=acc_idx for next iteration */ 
                acc_idx[level] = idx + 1; 
                get_idx[level] = idx + mask*2; 
                idx            = idx + 1; 
            } 
            else { 
                /* go to right for acc_idx, go to left for
                   get_idx. set idx=acc_idx for next iteration */ 
                acc_idx[level] = idx + mask*2; 
                get_idx[level] = idx + 1; 
                idx            = idx + mask*2; 
            } 
            level++;
            tmp_rank = tmp_rank % mask;
            mask >>= 1; 
        } 

/*        for (i=0; i<nlevels; i++)
            printf("Rank %d, acc_idx[%d]=%d, get_idx[%d]=%d\n", rank,
                   i, acc_idx[i], i, get_idx[i]);
*/

        MPI_Type_create_indexed_block(nlevels, 1, get_idx, MPI_INT, &get_type);
        MPI_Type_create_indexed_block(nlevels, 1, acc_idx, MPI_INT, &acc_type);
        MPI_Type_commit(&get_type);
        MPI_Type_commit(&acc_type);

        /* allocate array to store the values obtained from the 
           fetch-and-add counter */
        counter_vals = (int *) malloc(NTIMES * sizeof(int));

        MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); 

        for (i=0; i<NTIMES; i++) {
            Get_nextval_tree(win, get_array, get_type, acc_type,
                             nlevels, counter_vals+i); 
            /* printf("Rank %d, counter %d\n", rank, value); */
        }

        MPI_Win_free(&win);
        free(get_array);
        free(get_idx);
        free(acc_idx);
        MPI_Type_free(&get_type);
        MPI_Type_free(&acc_type);

         /* gather the results to the root */
        MPI_Gather(counter_vals, NTIMES, MPI_INT, NULL, 0, MPI_DATATYPE_NULL, 
                   0, MPI_COMM_WORLD);
        free(counter_vals);
   }

    MTest_Finalize(errs);
    MPI_Finalize(); 
    return MTestReturnValue( errs );
} 

int main(int argc, char* argv[]) {
    assert(argc == 8);

    MPI_Init(NULL, NULL);
    MPI_Datatype mpi_spike = create_spike_type();
    int rank, size;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    int ngroups = atoi(argv[1]);
    int simtime = atoi(argv[2]);
    int ncells = atoi(argv[3]);
    int fanin = atoi(argv[4]);
    int nSpikes = atoi(argv[5]);
    int mindelay = atoi(argv[6]);
    bool algebra = atoi(argv[7]);

    struct timeval start, end;

    int cellsper = ncells / size;

    //create environment
    environment::event_generator generator(ngroups);

    double mean = static_cast<double>(simtime) / static_cast<double>(nSpikes);
    double lambda = 1.0 / static_cast<double>(mean * size);

    environment::continousdistribution neuro_dist(size, rank, ncells);

    environment::generate_events_kai(generator.begin(),
                              simtime, ngroups, rank, size, lambda, &neuro_dist);

    environment::presyn_maker presyns(fanin);
    presyns(rank, &neuro_dist);
    spike::spike_interface s_interface(size);

    //run simulation
    MPI_Comm neighborhood = create_dist_graph(presyns, cellsper);
    queueing::pool pl(algebra, ngroups, mindelay, rank, s_interface);
    gettimeofday(&start, NULL);
    while(pl.get_time() <= simtime){
        pl.fixed_step(generator, presyns);
        distributed_spike(s_interface, mpi_spike, neighborhood);
        pl.filter(presyns);
    }
    gettimeofday(&end, NULL);

    long long diff_ms = (1000 * (end.tv_sec - start.tv_sec))
        + ((end.tv_usec - start.tv_usec) / 1000);

    if(rank == 0){
        std::cout<<"run time: "<<diff_ms<<" ms"<<std::endl;
    }

    pl.accumulate_stats();
    accumulate_stats(s_interface);

    MPI_Comm_free(&neighborhood);
    MPI_Type_free(&mpi_spike);
    MPI_Finalize();
    return 0;
}

int TestVecPackDouble(int n, int stride,
                      double *avgTimeUser, double *avgTimeMPI, double *dest, const double *src)
{
    double *restrict d_dest;
    const double *restrict d_src;
    register int i, j;
    int rep, position;
    double t1, t2, t[NTRIALS];
    MPI_Datatype vectype;

    /* User code */
    if (verbose)
        printf("TestVecPackDouble (USER): ");
    for (j = 0; j < NTRIALS; j++) {
        t1 = MPI_Wtime();
        for (rep = 0; rep < N_REPS; rep++) {
            i = n;
            d_dest = dest;
            d_src = src;
            while (i--) {
                *d_dest++ = *d_src;
                d_src += stride;
            }
        }
        t2 = MPI_Wtime() - t1;
        t[j] = t2;
        if (verbose)
            printf("%.3f ", t[j]);
    }
    if (verbose)
        printf("[%.3f]\n", noise(t, NTRIALS));
    /* If there is too much noise, discard the test */
    if (noise(t, NTRIALS) > VARIANCE_THRESHOLD) {
        *avgTimeUser = 0;
        *avgTimeMPI = 0;
        if (verbose)
            printf("Too much noise; discarding measurement\n");
        return 0;
    }
    *avgTimeUser = mean(t, NTRIALS) / N_REPS;

    /* MPI Vector code */
    MPI_Type_vector(n, 1, stride, MPI_DOUBLE, &vectype);
    MPI_Type_commit(&vectype);

    if (verbose)
        printf("TestVecPackDouble (MPI): ");
    for (j = 0; j < NTRIALS; j++) {
        t1 = MPI_Wtime();
        for (rep = 0; rep < N_REPS; rep++) {
            position = 0;
            MPI_Pack((void *) src, 1, vectype, dest, n * sizeof(double), &position, MPI_COMM_SELF);
        }
        t2 = MPI_Wtime() - t1;
        t[j] = t2;
        if (verbose)
            printf("%.3f ", t[j]);
    }
    if (verbose)
        printf("[%.3f]\n", noise(t, NTRIALS));
    /* If there is too much noise, discard the test */
    if (noise(t, NTRIALS) > VARIANCE_THRESHOLD) {
        *avgTimeUser = 0;
        *avgTimeMPI = 0;
        if (verbose)
            printf("Too much noise; discarding measurement\n");
    }
    else {
        *avgTimeMPI = mean(t, NTRIALS) / N_REPS;
    }

    MPI_Type_free(&vectype);

    return 0;
}

int main(int argc, char * argv[])
{
    int provided;
    MPI_ASSERT(MPI_Init_thread(&argc, &argv, MPI_THREAD_SINGLE, &provided));

    int rank, size;
    MPI_ASSERT(MPI_Comm_rank(MPI_COMM_WORLD, &rank));
    MPI_ASSERT(MPI_Comm_size(MPI_COMM_WORLD, &size));

    int logn = (argc>1) ? atoi(argv[1]) : 32;
    size_t count = (size_t)1<<logn; /* explicit cast required */
    printf("count = %zu \n", count );

    MPI_Datatype bigtype;
    MPI_ASSERT(MPIX_Type_contiguous_x( (MPI_Count)count, MPI_CHAR, &bigtype));
    MPI_ASSERT(MPI_Type_commit(&bigtype));

    char * rbuf = NULL;
    char * sbuf = NULL;

#ifdef USE_MPI_ALLOC_MEM
    MPI_ASSERT(MPI_Alloc_mem( (MPI_Aint)count * sizeof(char), MPI_INFO_NULL, &rbuf));
    MPI_ASSERT(MPI_Alloc_mem( (MPI_Aint)count * sizeof(char), MPI_INFO_NULL, &sbuf));
#else
    rbuf = malloc( count * sizeof(char));
    assert(rbuf!=NULL);
    sbuf = malloc( count * sizeof(char));
    assert(sbuf!=NULL);
#endif

    for (size_t i=0; i<count; i++)
        rbuf[i] = 'a';
    for (size_t i=0; i<count; i++)
        sbuf[i] = 'z';

    MPI_Request requests[2];
    MPI_Status statuses[2];

    if (rank==(size-1)) {
        MPI_ASSERT(MPI_Irecv(rbuf, 1, bigtype, 0,      0, MPI_COMM_WORLD, &(requests[1]) ));
    }
    if (rank==0) {
        MPI_ASSERT(MPI_Isend(sbuf, 1, bigtype, size-1, 0, MPI_COMM_WORLD, &(requests[0]) ));
    }

    MPI_Count ocount[2];

    if (size==1) {
        MPI_ASSERT(MPI_Waitall(2, requests, statuses));
        MPI_ASSERT(MPI_Get_elements_x( &(statuses[1]), MPI_CHAR, &(ocount[1])));
    }
    else {
        if (rank==(size-1)) {
            MPI_ASSERT(MPI_Wait( &(requests[1]), &(statuses[1]) ));
            MPI_ASSERT(MPI_Get_elements_x( &(statuses[1]), MPI_CHAR, &(ocount[1]) ));
        } else if (rank==0) {
            MPI_ASSERT(MPI_Wait( &(requests[0]), &(statuses[0]) ));
            MPI_ASSERT(MPI_Get_elements_x( &(statuses[0]), MPI_CHAR, &(ocount[0]) ));
        }
    }

    if (rank==0) {
        printf("ocount[0] = %lld \n", ocount[0]);
    } else if ( rank==(size-1) ) {
        printf("ocount[1] = %lld \n", ocount[1]);
    }

    /* correctness check */
    if (rank==(size-1)) {
        MPI_Count errors = 0;
        for (MPI_Count i=0; i<count; i++)
            errors += ( rbuf[i] != 'z' );
        printf("errors = %lld \n", errors);
    }

#ifdef USE_MPI_ALLOC_MEM
    MPI_ASSERT(MPI_Free_mem(rbuf));
    MPI_ASSERT(MPI_Free_mem(sbuf));
#else
    free(rbuf);
    free(sbuf);
#endif

    MPI_ASSERT(MPI_Type_free(&bigtype));

    MPI_ASSERT(MPI_Finalize());

    return 0;
}

int main (int argc, char *argv[]) {
    if(argc<3)
    {
      //  printf("Usage: ./executables/mandelbrot <rows> <cols> (filename)\n");
        return 0;
    }

    int rank, numprocs, err=0, i, j;
    std::vector<std::string> v;
    rows = atoi(argv[1]);
    cols = atoi(argv[2]);
    Matrix<double> img(rows, cols);
    std::string filename;
    if(argc > 3)
    {
        filename = argv[3];
        filename += ".ppm";
    }

    std::chrono::high_resolution_clock::time_point start, end;
    std::chrono::duration<double> time_span;
 
    start = std::chrono::high_resolution_clock::now();
    // Init MPI & get numprocs and rank
    MPI_Init(&argc,&argv);
    err = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    err = MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
    if(err){
        fprintf(stderr,"Catastrophic MPI problem.\n");
        MPI_Abort(MPI_COMM_WORLD,1);
    }

    Construct_MPI_Datatypes<double>(rows, cols);
   
    if(rank == 0)
    {
        master<double>(numprocs, img);
    }
    slave<double>(numprocs);

    std::cout<<"rank " <<rank <<" waiting at barrier\n";
    MPI_Barrier(MPI_COMM_WORLD);

    if(rank == 0)
    {
        end = std::chrono::high_resolution_clock::now();
        time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
        printf("\nProgram exec time %dx%d image: %f seconds \n\n",rows, cols, (double)time_span.count());

        if(argc > 3)
        {
            std::ofstream fout;
            fout.open(filename, std::ofstream::out | std::ofstream::binary);
            fout << "P3\n # Mandelbrot\n " << rows << " " <<cols << "\n 255\n";
             double val;
            int r, g, b;
            for(i=0; i<=std::floor(rows/2.0); i++)
            {
                for(j=0; j<cols; j++)
                {
                    val = std::abs(img[i][j]);
                    r = val * Palette[(int)val % 5][0]; 
                    g = val * Palette[(int)val % 5][1]; 
                    b = val * Palette[(int)val % 5][2]; 
                    fout << r << " " << g <<" " << b << " ";
                }
                fout <<"\n";
            }
            for(i=std::floor(rows/2.0); i>=0; i--)
            {
                for(j=0; j<cols; j++)
                {
                    val = std::abs(img[i][j]);
                    r = val * Palette[(int)val % 5][0]; 
                    g = val * Palette[(int)val % 5][1]; 
                    b = val * Palette[(int)val % 5][2]; 
                    fout << r << " " << g <<" " << b << " ";
                }
                fout <<"\n";
            }
            fout.close();
        }
    }

    // Free memory
    MPI_Type_free(&MPI_Vector);

    MPI_Finalize();
    return 0;
}

/* builtin_struct_test()
 *
 * Tests behavior with a zero-count struct of builtins.
 *
 * Returns the number of errors encountered.
 */
int builtin_struct_test(void)
{
    int err, errs = 0;

    int count = 0;
    MPI_Datatype newtype;

    int size;
    MPI_Aint extent;

    err = MPI_Type_create_struct(count,
				 (int *) 0,
				 (MPI_Aint *) 0,
				 (MPI_Datatype *) 0,
				 &newtype);
    if (err != MPI_SUCCESS) {
	if (verbose) {
	    fprintf(stderr,
		    "error creating struct type in builtin_struct_test()\n");
	}
	errs++;
    }

    err = MPI_Type_size(newtype, &size);
    if (err != MPI_SUCCESS) {
	if (verbose) {
	    fprintf(stderr,
		    "error obtaining type size in builtin_struct_test()\n");
	}
	errs++;
    }
    
    if (size != 0) {
	if (verbose) {
	    fprintf(stderr,
		    "error: size != 0 in builtin_struct_test()\n");
	}
	errs++;
    }    

    err = MPI_Type_extent(newtype, &extent);
    if (err != MPI_SUCCESS) {
	if (verbose) {
	    fprintf(stderr,
		    "error obtaining type extent in builtin_struct_test()\n");
	}
	errs++;
    }
    
    if (extent != 0) {
	if (verbose) {
	    fprintf(stderr,
		    "error: extent != 0 in builtin_struct_test()\n");
	}
	errs++;
    }    

    MPI_Type_free( &newtype );

    return errs;
}

/* optimizable_vector_of_basics_test()
 *
 * Builds a vector of ints.  Count is 10, blocksize is 2, stride is 2, so this
 * is equivalent to a contig of 20.  But remember...we should get back our
 * suboptimal values under MPI-2.
 *
 * Returns the number of errors encountered.
 */
int optimizable_vector_of_basics_test(void)
{
    MPI_Datatype parent_type;

    int nints, nadds, ntypes, combiner, *ints;
    MPI_Aint *adds = NULL;
    MPI_Datatype *types;

    int err, errs = 0;

    /* set up type */
    err = MPI_Type_vector(10,
			  2,
			  2,
			  MPI_INT,
			  &parent_type);

    /* decode */
    err = MPI_Type_get_envelope(parent_type,
				&nints,
				&nadds,
				&ntypes,
				&combiner);

    if (nints != 3) errs++;
    if (nadds != 0) errs++;
    if (ntypes != 1) errs++;
    if (combiner != MPI_COMBINER_VECTOR) errs++;

    if (verbose) {
        if (nints != 3) fprintf(stderr, "nints = %d; should be 3\n", nints);
	if (nadds != 0) fprintf(stderr, "nadds = %d; should be 0\n", nadds);
	if (ntypes != 1) fprintf(stderr, "ntypes = %d; should be 1\n", ntypes);
	if (combiner != MPI_COMBINER_VECTOR)
	    fprintf(stderr, "combiner = %s; should be vector\n",
		    combiner_to_string(combiner));
    }

    ints = malloc(nints * sizeof(*ints));
    if (nadds) adds = malloc(nadds * sizeof(*adds));
    types = malloc(ntypes *sizeof(*types));

    err = MPI_Type_get_contents(parent_type,
				nints,
				nadds,
				ntypes,
				ints,
				adds,
				types);

    if (ints[0] != 10) errs++;
    if (ints[1] != 2) errs++;
    if (ints[2] != 2) errs++;
    if (types[0] != MPI_INT) errs++;

    if (verbose) {
	if (ints[0] != 10) fprintf(stderr, "count = %d; should be 10\n",
				   ints[0]);
	if (ints[1] != 2) fprintf(stderr, "blocklength = %d; should be 2\n",
				  ints[1]);
	if (ints[2] != 2) fprintf(stderr, "stride = %d; should be 2\n",
				  ints[2]);
	if (types[0] != MPI_INT) fprintf(stderr, "type is not MPI_INT\n");
    }

    free(ints);
    if (nadds) free(adds);
    free(types);

    MPI_Type_free( &parent_type );

    return errs;
}

/* indexed_of_basics_test(void)
 *
 * Simple indexed type.
 *
 * Returns number of errors encountered.
 */
int indexed_of_basics_test(void)
{
    MPI_Datatype parent_type;
    int s_count = 3, s_blocklengths[3] = { 3, 2, 1 };
    int s_displacements[3] = { 10, 20, 30 };

    int nints, nadds, ntypes, combiner, *ints;
    MPI_Aint *adds = NULL;
    MPI_Datatype *types;

    int err, errs = 0;

    /* set up type */
    err = MPI_Type_indexed(s_count,
			   s_blocklengths,
			   s_displacements,
			   MPI_INT,
			   &parent_type);

    /* decode */
    err = MPI_Type_get_envelope(parent_type,
				&nints,
				&nadds,
				&ntypes,
				&combiner);

    if (nints != 7) errs++;
    if (nadds != 0) errs++;
    if (ntypes != 1) errs++;
    if (combiner != MPI_COMBINER_INDEXED) errs++;

    if (verbose) {
        if (nints != 7) fprintf(stderr, "nints = %d; should be 7\n", nints);
	if (nadds != 0) fprintf(stderr, "nadds = %d; should be 0\n", nadds);
	if (ntypes != 1) fprintf(stderr, "ntypes = %d; should be 1\n", ntypes);
	if (combiner != MPI_COMBINER_INDEXED)
	    fprintf(stderr, "combiner = %s; should be indexed\n",
		    combiner_to_string(combiner));
    }

    ints = malloc(nints * sizeof(*ints));
    if (nadds) adds = malloc(nadds * sizeof(*adds));
    types = malloc(ntypes *sizeof(*types));

    err = MPI_Type_get_contents(parent_type,
				nints,
				nadds,
				ntypes,
				ints,
				adds,
				types);

    if (ints[0] != s_count) errs++;
    if (ints[1] != s_blocklengths[0]) errs++;
    if (ints[2] != s_blocklengths[1]) errs++;
    if (ints[3] != s_blocklengths[2]) errs++;
    if (ints[4] != s_displacements[0]) errs++;
    if (ints[5] != s_displacements[1]) errs++;
    if (ints[6] != s_displacements[2]) errs++;
    if (types[0] != MPI_INT) errs++;

    if (verbose) {
	if (ints[0] != s_count) 
	    fprintf(stderr, "count = %d; should be %d\n", ints[0], s_count);
	if (ints[1] != s_blocklengths[0]) 
	    fprintf(stderr, "blocklength[0] = %d; should be %d\n", ints[1], s_blocklengths[0]);
	if (ints[2] != s_blocklengths[1]) 
	    fprintf(stderr, "blocklength[1] = %d; should be %d\n", ints[2], s_blocklengths[1]);
	if (ints[3] != s_blocklengths[2]) 
	    fprintf(stderr, "blocklength[2] = %d; should be %d\n", ints[3], s_blocklengths[2]);
	if (ints[4] != s_displacements[0]) 
	    fprintf(stderr, "displacement[0] = %d; should be %d\n", ints[4], s_displacements[0]);
	if (ints[5] != s_displacements[1]) 
	    fprintf(stderr, "displacement[1] = %d; should be %d\n", ints[5], s_displacements[1]);
	if (ints[6] != s_displacements[2]) 
	    fprintf(stderr, "displacement[2] = %d; should be %d\n", ints[6], s_displacements[2]);
	if (types[0] != MPI_INT) fprintf(stderr, "type[0] does not match\n");
    }

    free(ints);
    if (nadds) free(adds);
    free(types);

    MPI_Type_free( &parent_type );
    return errs;
}

//.........这里部分代码省略.........
		ADIOI_Free(old_pvfs_dtype);
		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
			"F90_COMPLEX is unsupported\n");
		break;
	    case MPI_COMBINER_F90_INTEGER:
		ADIOI_Free(old_pvfs_dtype);
		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
			"F90_INTEGER is unsupported\n");
		break;
	    case MPI_COMBINER_RESIZED:
		ADIOI_Free(old_pvfs_dtype);
		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
			"RESIZED is unsupported\n");
		break;
	    default:
		break;
	}

	if (ret != 0)
	    fprintf(stderr, "Error in PVFS_Request_* "
		    "for a derived datatype\n");

#ifdef DEBUG_DTYPE
	print_dtype_info(combiner,
			 num_int,
			 num_addr,
			 num_dtype,
			 arr_int,
			 arr_addr,
			 arr_dtype);
#endif

	if (leaf != 1 && combiner != MPI_COMBINER_DUP)
	    MPI_Type_free(&arr_dtype[0]);

	ADIOI_Free(arr_int);
	ADIOI_Free(arr_addr);
	ADIOI_Free(arr_dtype);

	PVFS_Request_free(old_pvfs_dtype);
	ADIOI_Free(old_pvfs_dtype);

	return ret;
    }
    else /* MPI_COMBINER_STRUCT */
    {
	MPI_Aint mpi_lb = -1, mpi_extent = -1;
	PVFS_offset pvfs_lb = -1;
	PVFS_size pvfs_extent = -1;
	int has_lb_ub = 0;

	/* When converting into a PVFS_Request_struct, we no longer
	 * can use MPI_LB and MPI_UB.  Therfore, we have to do the
	 * following.
	 * We simply ignore all the MPI_LB and MPI_UB types and
	 * get the lb and extent and pass it on through a
	 * PVFS resized_req */

	arr_count = 0;
	for (i = 0; i < arr_int[0]; i++)
	{
	    if (arr_dtype[i] != MPI_LB &&
		arr_dtype[i] != MPI_UB)
	    {
		arr_count++;
	    }

//.........这里部分代码省略.........

    if (verbose) {
	if (ints[0] != 2) fprintf(stderr, 
				  "outer vector count = %d; should be 2\n",
				  ints[0]);
	if (ints[1] != 1) fprintf(stderr,
				  "outer vector blocklength = %d; should be 1\n",
				  ints[1]);
	if (ints[2] != 2) fprintf(stderr, "outer vector stride = %d; should be 2\n",
				  ints[2]);
    }
    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    inner_vector_copy = types[0];
    free(ints);
    if (nadds) free(adds);
    free(types);

    /* decode inner vector */
    err = MPI_Type_get_envelope(inner_vector_copy,
				&nints,
				&nadds,
				&ntypes,
				&combiner);
    if (err != MPI_SUCCESS) {
	if (verbose) fprintf(stderr, 
			     "error in MPI call; aborting after %d errors\n",
			     errs+1);
	return errs+1;
    }

    if (nints != 3) errs++;
    if (nadds != 0) errs++;
    if (ntypes != 1) errs++;
    if (combiner != MPI_COMBINER_VECTOR) errs++;

    if (verbose) {
	if (nints != 3) fprintf(stderr, 
				"inner vector nints = %d; should be 3\n",
				nints);
	if (nadds != 0) fprintf(stderr, 
				"inner vector nadds = %d; should be 0\n",
				nadds);
	if (ntypes != 1) fprintf(stderr, 
				 "inner vector ntypes = %d; should be 1\n",
				 ntypes);
	if (combiner != MPI_COMBINER_VECTOR)
	    fprintf(stderr, "inner vector combiner = %s; should be vector\n",
		    combiner_to_string(combiner));
    }
    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    ints = malloc(nints * sizeof(*ints));
    if (nadds) adds = malloc(nadds * sizeof(*adds));
    types = malloc(ntypes * sizeof(*types));

    err = MPI_Type_get_contents(inner_vector_copy,
				nints,
				nadds,
				ntypes,
				ints,
				adds,
				types);

    if (ints[0] != 2) errs++;
    if (ints[1] != 1) errs++;
    if (ints[2] != 2) errs++;

    if (verbose) {
	if (ints[0] != 2) fprintf(stderr, 
				  "inner vector count = %d; should be 2\n",
				  ints[0]);
	if (ints[1] != 1) fprintf(stderr,
				  "inner vector blocklength = %d; should be 1\n",
				  ints[1]);
	if (ints[2] != 2) fprintf(stderr, 
				  "inner vector stride = %d; should be 2\n",
				  ints[2]);
    }
    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    free(ints);
    if (nadds) free(adds);
    free(types);

    MPI_Type_free( &inner_vector_copy );
    MPI_Type_free( &inner_vector );
    MPI_Type_free( &outer_vector );

    return 0;
}

int amps_unpack(
   amps_Comm comm,
   amps_Invoice inv,
   char *buffer,
   int buf_size)
{
   amps_InvoiceEntry *ptr;
   int len, stride;
   int  malloced = FALSE;
   int size;
   char *data;
   char *temp_pos;
   int dim;

   MPI_Datatype mpi_type;
   MPI_Datatype *base_type;
   MPI_Datatype *new_type;
   MPI_Datatype *temp_type;

   int element_size = 0;
   int position;

   int base_size;

   int i;
   
   /* we are unpacking so signal this operation */
   
   inv -> flags &= ~AMPS_PACKED;
   
   /* for each entry in the invoice pack that entry into the letter         */
   ptr = inv -> list;
   position = 0;
   while(ptr != NULL)
   {
      /* invoke the packing convert out for the entry */
      /* if user then call user ones */
      /* else switch on builtin type */
      if(ptr -> len_type == AMPS_INVOICE_POINTER)
	 len = *(ptr -> ptr_len);
      else
	 len = ptr ->len;
      
      if(ptr -> stride_type == AMPS_INVOICE_POINTER)
	 stride = *(ptr -> ptr_stride);
      else
	 stride = ptr -> stride;
      
      switch(ptr->type)
      {
      case AMPS_INVOICE_CHAR_CTYPE:
	 if(!ptr->ignore)
	 {
	    if( ptr -> data_type == AMPS_INVOICE_POINTER)
	    {
	       *((void **)(ptr -> data)) = malloc(sizeof(char) *
						  (size_t)(len*stride));
	       malloced = TRUE;

	       MPI_Type_vector(len, 1, stride, MPI_BYTE, &mpi_type);

	       MPI_Type_commit(&mpi_type);

	       MPI_Unpack(buffer, buf_size, &position, 
			  *((void **)(ptr -> data)), 1, mpi_type, comm);

	       MPI_Type_free(&mpi_type);
	    }
	    else
	    {
	       MPI_Type_vector(len, 1, stride, MPI_BYTE, &mpi_type);

	       MPI_Type_commit(&mpi_type);
	       MPI_Unpack(buffer, buf_size, &position, 
			  ptr -> data, 1, mpi_type, comm);
	       MPI_Type_free(&mpi_type);
	    }
	 }
	 break;
      case AMPS_INVOICE_SHORT_CTYPE:
	 if(!ptr->ignore)
	 {
	    if( ptr -> data_type == AMPS_INVOICE_POINTER)
	    {
	       *((void **)(ptr -> data)) = malloc(sizeof(short) * 
						  (size_t)(len*stride));
	       malloced = TRUE;

	       MPI_Type_vector(len, 1, stride, MPI_SHORT, &mpi_type);

	       MPI_Type_commit(&mpi_type);
	       MPI_Unpack(buffer, buf_size, &position, 
			  *((void **)(ptr -> data)), 1, mpi_type, comm);
	       MPI_Type_free(&mpi_type);
	    }
	    else
	    {
	       MPI_Type_vector(len, 1, stride, MPI_SHORT, &mpi_type);

	       MPI_Type_commit(&mpi_type);
//.........这里部分代码省略.........

//.........这里部分代码省略.........
  }

  {
    char fname[PETSC_MAX_PATH_LEN];
    FILE *fd = NULL;

    fname[0] = 0;

    ierr = PetscOptionsGetString(NULL,"-malloc_log",fname,250,&flg1);CHKERRQ(ierr);
    ierr = PetscOptionsHasName(NULL,"-malloc_log_threshold",&flg2);CHKERRQ(ierr);
    if (flg1 && fname[0]) {
      int err;

      if (!rank) {
        fd = fopen(fname,"w");
        if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",fname);
      }
      ierr = PetscMallocDumpLog(fd);CHKERRQ(ierr);
      if (fd) {
        err = fclose(fd);
        if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
      }
    } else if (flg1 || flg2) {
      ierr = PetscMallocDumpLog(stdout);CHKERRQ(ierr);
    }
  }
  /* Can be destroyed only after all the options are used */
  ierr = PetscOptionsDestroy();CHKERRQ(ierr);

  PetscGlobalArgc = 0;
  PetscGlobalArgs = 0;

#if defined(PETSC_USE_REAL___FLOAT128)
  ierr = MPI_Type_free(&MPIU___FLOAT128);CHKERRQ(ierr);
#if defined(PETSC_HAVE_COMPLEX)
  ierr = MPI_Type_free(&MPIU___COMPLEX128);CHKERRQ(ierr);
#endif
  ierr = MPI_Op_free(&MPIU_MAX);CHKERRQ(ierr);
  ierr = MPI_Op_free(&MPIU_MIN);CHKERRQ(ierr);
#endif

#if defined(PETSC_HAVE_COMPLEX)
#if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
  ierr = MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
  ierr = MPI_Type_free(&MPIU_C_COMPLEX);CHKERRQ(ierr);
#endif
#endif

#if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
  ierr = MPI_Op_free(&MPIU_SUM);CHKERRQ(ierr);
#endif

  ierr = MPI_Type_free(&MPIU_2SCALAR);CHKERRQ(ierr);
#if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
  ierr = MPI_Type_free(&MPIU_2INT);CHKERRQ(ierr);
#endif
  ierr = MPI_Op_free(&PetscMaxSum_Op);CHKERRQ(ierr);
  ierr = MPI_Op_free(&PetscADMax_Op);CHKERRQ(ierr);
  ierr = MPI_Op_free(&PetscADMin_Op);CHKERRQ(ierr);

  /*
     Destroy any known inner MPI_Comm's and attributes pointing to them
     Note this will not destroy any new communicators the user has created.

     If all PETSc objects were not destroyed those left over objects will have hanging references to
     the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again

int main(int argc, char **argv) {
    int i, j, rank, nranks, peer, bufsize, errors;
    double *win_buf, *src_buf;
    MPI_Win buf_win;

    MTest_Init(&argc, &argv);

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nranks);

    bufsize = XDIM * YDIM * sizeof(double);
    MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &win_buf);
    MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &src_buf);

    if (rank == 0)
        if (verbose) printf("MPI RMA Strided Accumulate Test:\n");

    for (i = 0; i < XDIM*YDIM; i++) {
        *(win_buf + i) = -1.0;
        *(src_buf + i) = 1.0 + rank;
    }

    MPI_Win_create(win_buf, bufsize, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &buf_win);

    peer = (rank+1) % nranks;

    /* Perform ITERATIONS strided accumulate operations */

    for (i = 0; i < ITERATIONS; i++) {
      int ndims               = 2;
      int src_arr_sizes[2]    = { XDIM, YDIM };
      int src_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM };
      int src_arr_starts[2]   = {    0,    0 };
      int dst_arr_sizes[2]    = { XDIM, YDIM };
      int dst_arr_subsizes[2] = { SUB_XDIM, SUB_YDIM };
      int dst_arr_starts[2]   = {    0,    0 };
      MPI_Datatype src_type, dst_type;

      MPI_Type_create_subarray(ndims, src_arr_sizes, src_arr_subsizes, src_arr_starts,
          MPI_ORDER_C, MPI_DOUBLE, &src_type);

      MPI_Type_create_subarray(ndims, dst_arr_sizes, dst_arr_subsizes, dst_arr_starts,
          MPI_ORDER_C, MPI_DOUBLE, &dst_type);

      MPI_Type_commit(&src_type);
      MPI_Type_commit(&dst_type);

      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win);

      MPI_Accumulate(src_buf, 1, src_type, peer, 0, 1, dst_type, MPI_SUM, buf_win);

      MPI_Win_unlock(peer, buf_win);

      MPI_Type_free(&src_type);
      MPI_Type_free(&dst_type);
    }

    MPI_Barrier(MPI_COMM_WORLD);

    /* Verify that the results are correct */

    MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win);
    errors = 0;
    for (i = 0; i < SUB_XDIM; i++) {
      for (j = 0; j < SUB_YDIM; j++) {
        const double actual   = *(win_buf + i + j*XDIM);
        const double expected = -1.0 + (1.0 + ((rank+nranks-1)%nranks)) * (ITERATIONS);
        if (fabs(actual - expected) > 1.0e-10) {
          SQUELCH( printf("%d: Data validation failed at [%d, %d] expected=%f actual=%f\n",
              rank, j, i, expected, actual); );
          errors++;
          fflush(stdout);
        }
      }

void ADIOI_HFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
{
    MPI_Datatype copy_etype, copy_filetype;
    int combiner, i, j, k, filetype_is_contig, ntimes, err;
    ADIOI_Flatlist_node *flat_file;
    ADIO_Offset curr_fsize, alloc_size, size, len, done;
    ADIO_Status status;
    char *buf;
#ifndef PRINT_ERR_MSG
    static char myname[] = "ADIOI_HFS_FCNTL";
#endif

    switch(flag) {
    case ADIO_FCNTL_SET_VIEW:
        /* free copies of old etypes and filetypes and delete flattened 
           version of filetype if necessary */

	MPI_Type_get_envelope(fd->etype, &i, &j, &k, &combiner);
	if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->etype));

	ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
	if (!filetype_is_contig) ADIOI_Delete_flattened(fd->filetype);

	MPI_Type_get_envelope(fd->filetype, &i, &j, &k, &combiner);
	if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->filetype));

	/* set new info */
	ADIO_SetInfo(fd, fcntl_struct->info, &err);

        /* set new etypes and filetypes */

	MPI_Type_get_envelope(fcntl_struct->etype, &i, &j, &k, &combiner);
	if (combiner == MPI_COMBINER_NAMED) fd->etype = fcntl_struct->etype;
	else {
	    MPI_Type_contiguous(1, fcntl_struct->etype, &copy_etype);
	    MPI_Type_commit(&copy_etype);
	    fd->etype = copy_etype;
	}
	MPI_Type_get_envelope(fcntl_struct->filetype, &i, &j, &k, &combiner);
	if (combiner == MPI_COMBINER_NAMED) 
	    fd->filetype = fcntl_struct->filetype;
	else {
	    MPI_Type_contiguous(1, fcntl_struct->filetype, &copy_filetype);
	    MPI_Type_commit(&copy_filetype);
	    fd->filetype = copy_filetype;
	    ADIOI_Flatten_datatype(fd->filetype);
            /* this function will not flatten the filetype if it turns out
               to be all contiguous. */
	}

	MPI_Type_size(fd->etype, &(fd->etype_size));
	fd->disp = fcntl_struct->disp;

        /* reset MPI-IO file pointer to point to the first byte that can
           be accessed in this view. */

        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
	if (filetype_is_contig) fd->fp_ind = fcntl_struct->disp;
	else {
	    flat_file = ADIOI_Flatlist;
	    while (flat_file->type != fd->filetype) 
		flat_file = flat_file->next;
	    for (i=0; i<flat_file->count; i++) {
		if (flat_file->blocklens[i]) {
		    fd->fp_ind = fcntl_struct->disp + flat_file->indices[i];
		    break;
		}
	    }
	}
	*error_code = MPI_SUCCESS;
	break;

    case ADIO_FCNTL_GET_FSIZE:
	fcntl_struct->fsize = lseek64(fd->fd_sys, 0, SEEK_END);
#ifdef PRINT_ERR_MSG
	*error_code = (fcntl_struct->fsize == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
#else
    if (fcntl_struct->fsize == -1) {
	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
			      myname, "I/O Error", "%s", strerror(errno));
	ADIOI_Error(fd, *error_code, myname);	    
    }
    else *error_code = MPI_SUCCESS;
#endif
	break;

    case ADIO_FCNTL_SET_DISKSPACE:
	/* will be called by one process only */

#ifdef SPPUX
	/* SPPUX has no prealloc64. therefore, use prealloc
           if size < (2GB - 1), otherwise use long method. */
        if (fcntl_struct->diskspace <= 2147483647) {
	    err = prealloc(fd->fd_sys, (off_t) fcntl_struct->diskspace);
	    if (err && (errno != ENOTEMPTY)) {
#ifdef PRINT_ERR_MSG
    	        *error_code = MPI_ERR_UNKNOWN;
#else
		*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
			      myname, "I/O Error", "%s", strerror(errno));
//.........这里部分代码省略.........

//.........这里部分代码省略.........
	    fprintf(stderr, "blocklength[1] = %d; should be %d\n", ints[2], i_blocklengths[1]);
	if (ints[3] != i_blocklengths[2]) 
	    fprintf(stderr, "blocklength[2] = %d; should be %d\n", ints[3], i_blocklengths[2]);
	if (ints[4] != i_displacements[0]) 
	    fprintf(stderr, "displacement[0] = %d; should be %d\n", ints[4], i_displacements[0]);
	if (ints[5] != i_displacements[1]) 
	    fprintf(stderr, "displacement[1] = %d; should be %d\n", ints[5], i_displacements[1]);
	if (ints[6] != i_displacements[2]) 
	    fprintf(stderr, "displacement[2] = %d; should be %d\n", ints[6], i_displacements[2]);
    }

    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    inner_vector_copy = types[0];
    free(ints);
    if (nadds) free(adds);
    free(types);

    /* decode inner vector */
    err = MPI_Type_get_envelope(inner_vector_copy,
				&nints,
				&nadds,
				&ntypes,
				&combiner);
    if (err != MPI_SUCCESS) {
	if (verbose) fprintf(stderr, 
			     "error in MPI call; aborting after %d errors\n",
			     errs+1);
	return errs+1;
    }

    if (nints != 3) errs++;
    if (nadds != 0) errs++;
    if (ntypes != 1) errs++;
    if (combiner != MPI_COMBINER_VECTOR) errs++;

    if (verbose) {
	if (nints != 3) fprintf(stderr, 
				"inner vector nints = %d; should be 3\n",
				nints);
	if (nadds != 0) fprintf(stderr, 
				"inner vector nadds = %d; should be 0\n",
				nadds);
	if (ntypes != 1) fprintf(stderr, 
				 "inner vector ntypes = %d; should be 1\n",
				 ntypes);
	if (combiner != MPI_COMBINER_VECTOR)
	    fprintf(stderr, "inner vector combiner = %s; should be vector\n",
		    combiner_to_string(combiner));
    }
    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    ints = malloc(nints * sizeof(*ints));
    if (nadds) adds = malloc(nadds * sizeof(*adds));
    types = malloc(ntypes * sizeof(*types));

    err = MPI_Type_get_contents(inner_vector_copy,
				nints,
				nadds,
				ntypes,
				ints,
				adds,
				types);

    if (ints[0] != 2) errs++;
    if (ints[1] != 1) errs++;
    if (ints[2] != 2) errs++;

    if (verbose) {
	if (ints[0] != 2) fprintf(stderr, 
				  "inner vector count = %d; should be 2\n",
				  ints[0]);
	if (ints[1] != 1) fprintf(stderr,
				  "inner vector blocklength = %d; should be 1\n",
				  ints[1]);
	if (ints[2] != 2) fprintf(stderr, 
				  "inner vector stride = %d; should be 2\n",
				  ints[2]);
    }
    if (errs) {
	if (verbose) fprintf(stderr, "aborting after %d errors\n", errs);
	return errs;
    }

    free(ints);
    if (nadds) free(adds);
    free(types);

    MPI_Type_free( &inner_vector_copy );
    MPI_Type_free( &inner_vector );
    MPI_Type_free( &outer_indexed );

    return 0;
}