def make_csv(self, out_csv, array):
     if out_csv is None:
         return 0
     else:
         print "Generating csv"
         mlab.rec2csv(array, out_csv)
         return 1

def rec2csv(rec_array, csv_file, formatd=None, **kwargs):
    """
    Convenience wrapper function on top of mlab.rec2csv to allow fixed-
    precision output to CSV files

    Parameters
    ----------
    rec_aray : numpy 1-d recarray
        The recarray to be written out

    csv_file : str
        CSV file name

    kwargs : dict
        Keyword arguments to pass through to mlab.rec2csv

    Returns
    -------
    None
    """

    # Get the formatd objects associated with each field
    formatd = mlab.get_formatd(rec_array, formatd)

    # For all FormatFloat objects, switch to FormatDecimal objects
    for (k, v) in formatd.iteritems():
        if isinstance(v, mlab.FormatFloat):
            formatd[k] = FormatDecimal()

    # Pass this specification to mlab.rec2csv
    mlab.rec2csv(rec_array, csv_file, formatd=formatd, **kwargs)

def otherfunc(roifiles, subjects):
    import numpy as np
    from matplotlib.mlab import rec2csv
    import os

    first = np.recfromcsv(roifiles[0])
    numcons = len(first.dtype.names) - 1
    roinames = ["subject_id"] + first["roi"].tolist()
    formats = ["a20"] + ["f4" for f in roinames[1:]]
    confiles = []
    for con in range(0, numcons):
        recarray = np.zeros(len(roifiles), dtype={"names": roinames, "formats": formats})
        for i, file in enumerate(roifiles):
            recfile = np.recfromcsv(file)
            recarray["subject_id"][i] = subjects[i]
            for roi in roinames[1:]:
                value = recfile["con%02d" % (con + 1)][recfile["roi"] == roi]
                if value:
                    recarray[roi][i] = value
                else:
                    recarray[roi][i] = 999
        filename = os.path.abspath("grouped_con%02d.csv" % (con + 1))
        rec2csv(recarray, filename)
        confiles.append(filename)
    return confiles

def testR(d=simple(), size=500):

    X = random_from_categorical_formula(d, size)

    X = ML.rec_append_fields(X, 'response', np.random.standard_normal(size))
    fname = tempfile.mktemp()
    ML.rec2csv(X, fname)
    Rstr = '''
    data = read.table("%s", sep=',', header=T)
    cur.lm = lm(response ~ %s, data)
    COEF = coef(cur.lm)
    ''' % (fname, d.Rstr)
    rpy2.robjects.r(Rstr)
    remove(fname)
    nR = list(np.array(rpy2.robjects.r("names(COEF)")))

    nt.assert_true('(Intercept)' in nR)
    nR.remove("(Intercept)")
    nF = [str(t).replace("_","").replace("*",":") for t in d.formula.terms]
             
    nR = sorted([sorted(n.split(":")) for n in nR])

    nt.assert_true('1' in nF)
    nF.remove('1')

    nF = sorted([sorted(n.split(":")) for n in nF])
    nt.assert_equal(nR, nF)

    return d, X, nR, nF

def main():
    print "initializing"
    ap.env.overwriteOutput = True
    ap.env.workspace = WORKSPACE
    
    ras = ["marginal_ag_land_ha",
            "favored_ag_land_ha",
            "ag_wateronly_constrained_ha",
            "ag_landonly_constrained_ha",
            "ag_both_constrained_ha"]
    lbls = ["mar_ha","fav_ha","water_ha","land_ha","both_ha"]
    
    ap.CheckOutExtension("SPATIAL")
    
    POLYS = "mena_plus"
    POLYFIELD = "name"
    
    recs = []
    for i in range(len(ras)):
        ap.sa.ZonalStatisticsAsTable(POLYS,POLYFIELD,ras[i],lbls[i],"DATA","SUM")
        recs.append(ap.da.TableToNumPyArray(lbls[i],[POLYFIELD,"SUM"]))
    
    outrecs = [recs[i]["SUM"] for i in range(len(recs))]
    outrecs.extend([recs[i][POLYFIELD] for i in range(len(recs))])
    mlab.rec2csv(np.rec.fromarrays(outrecs, names=lbls),OUTCSV)
    
    
    print "complete"

def makediffs(models = _allmodels, verbose = False, kpp = True):
    for model in models:
        model = os.path.splitext(os.path.basename(model))[0]
        if kpp:
            kppdat = csv2rec(os.path.join(model, model + '.dat'), delimiter = ' ')
        else:
            if model not in _modelconfigs:
                raise IOError('If KPP is not properly installed, you cannot run tests on mechanisms other than cbm4, saprc99, and small_strato.')
            kppdat = csv2rec(os.path.join(os.path.dirname(__file__), model + '.dat'), delimiter = ' ')
        pykppdat = csv2rec(os.path.join(model, model + '.pykpp.dat'), delimiter = ',')
        diff = pykppdat.copy()
        pct = pykppdat.copy()
        keys = set(kppdat.dtype.names).intersection(pykppdat.dtype.names)
        notkeys = set(pykppdat.dtype.names).difference(kppdat.dtype.names)
        notkeys.remove('t')
        for k in notkeys:
            diff[k] = np.nan
            pct[k] = np.nan
    
        for k in keys:
            diff[k] = pykppdat[k] - kppdat[k][:]
            pct[k] = diff[k] / kppdat[k][:] * 100
        diff['t'] = pykppdat['t'] - (kppdat['time'] * 3600. + pykppdat['t'][0])
        pct['t'] = diff['t'] / (kppdat['time'] * 3600. + pykppdat['t'][0]) * 100
        
        rec2csv(diff, os.path.join(model, model + '.diff.csv'), delimiter = ',')
        rec2csv(pct, os.path.join(model, model + '.pct.csv'), delimiter = ',')

def rewrite_spec(subj, run, root = "/home/jtaylo/FIAC-HBM2009"):
    """
    Take a FIAC specification file and get two specifications
    (experiment, begin).

    This creates two new .csv files, one for the experimental
    conditions, the other for the "initial" confounding trials that
    are to be modelled out. 

    For the block design, the "initial" trials are the first
    trials of each block. For the event designs, the 
    "initial" trials are made up of just the first trial.

    """

    if exists(pjoin("%(root)s", "fiac%(subj)d", "subj%(subj)d_evt_fonc%(run)d.txt") % {'root':root, 'subj':subj, 'run':run}):
        designtype = 'evt'
    else:
        designtype = 'bloc'

    # Fix the format of the specification so it is
    # more in the form of a 2-way ANOVA

    eventdict = {1:'SSt_SSp', 2:'SSt_DSp', 3:'DSt_SSp', 4:'DSt_DSp'}
    s = StringIO()
    w = csv.writer(s)
    w.writerow(['time', 'sentence', 'speaker'])

    specfile = pjoin("%(root)s", "fiac%(subj)d", "subj%(subj)d_%(design)s_fonc%(run)d.txt") % {'root':root, 'subj':subj, 'run':run, 'design':designtype}
    d = np.loadtxt(specfile)
    for row in d:
        w.writerow([row[0]] + eventdict[row[1]].split('_'))
    s.seek(0)
    d = csv2rec(s)

    # Now, take care of the 'begin' event
    # This is due to the FIAC design

    if designtype == 'evt':
        b = np.array([(d[0]['time'], 1)], np.dtype([('time', np.float),
                                                    ('initial', np.int)]))
        d = d[1:]
    else:
        k = np.equal(np.arange(d.shape[0]) % 6, 0)
        b = np.array([(tt, 1) for tt in d[k]['time']], np.dtype([('time', np.float),
                                                                 ('initial', np.int)]))
        d = d[~k]

    designtype = {'bloc':'block', 'evt':'event'}[designtype]

    fname = pjoin(DATADIR, "fiac_%(subj)02d", "%(design)s", "experiment_%(run)02d.csv") % {'root':root, 'subj':subj, 'run':run, 'design':designtype}
    rec2csv(d, fname)
    experiment = csv2rec(fname)

    fname = pjoin(DATADIR, "fiac_%(subj)02d", "%(design)s", "initial_%(run)02d.csv") % {'root':root, 'subj':subj, 'run':run, 'design':designtype}
    rec2csv(b, fname)
    initial = csv2rec(fname)

    return d, b

    def to_file(self, filename, **kwargs):
        """
        Saves results to file, which will be gzipped if `filename` has a .gz
        extension.

        kwargs are passed to matplotlib.mlab.rec2csv
        """
        rec2csv(self.data, filename, **kwargs)

def test_rec2csv_bad_shape():
    try:
        bad = np.recarray((99,4),[('x',np.float),('y',np.float)])
        fd = tempfile.TemporaryFile(suffix='csv')
    
        # the bad recarray should trigger a ValueError for having ndim > 1.
        mlab.rec2csv(bad,fd)
    finally:
        fd.close()

def write_results_to_csv(results, directory):

    experiments, outcomes = results
#     deceased_pop = outcomes['relative market price']
#     time = outcomes[TIME]
    
    rec2csv(experiments, directory+'/experiments.csv', withheader=True)
    
    for key, value in outcomes.iteritems():
        np.savetxt(directory+'/{}.csv'.format(key), value, delimiter=',')

def interesting_out(opts,interesting,data):
    """
    Take a list of fields, and the recs
    output recs as csv to opts["out"], e.g. --out
    """
    header = True
    from matplotlib import mlab
    for d in data:
        cleaned = mlab.rec_keep_fields(d,interesting)
        mlab.rec2csv(cleaned,opts["out"],withheader=header)
        header=False

def main():
    inputlist = ["bin/global_BWS_20121015.csv","bin/global_WRI_20121015.csv"]
    lhs = mlab.csv2rec("bin/global_GU_20121015.csv")
    rhslist = []
    for x in inputlist:
        rhslist.append(mlab.csv2rec(x))
    
    rhslist[0]["basinid"] = rhslist[0]["basinid"].astype(np.long)
    keys = ("basinid","countryid","id")
    lhs = join_recs_on_keys(lhs,rhslist,keys)
    mlab.rec2csv(lhs,"bin/test.csv")
    print "complete"

def main():
    print "initializing"
    
    ap.env.overwriteOutput = True
    
    #"World_Cylindrical_Equal_Area"
    sr = ap.SpatialReference(54034) 
    ap.Project_management(BASINPOLY, TMP_OUT, sr)
    ap.CalculateAreas_stats(TMP_OUT,TMP_OUT2)
    out = ap.da.FeatureClassToNumPyArray(TMP_OUT2,[BASIN_ID_FIELD,"F_AREA"])
    mlab.rec2csv(out,AREACSV)
    
    print "complete"

def test_recarray_csv_roundtrip():
    expected = np.recarray((99,),
                          [('x',np.float),('y',np.float),('t',np.float)])
    expected['x'][:] = np.linspace(-1e9, -1, 99)
    expected['y'][:] = np.linspace(1, 1e9, 99)
    expected['t'][:] = np.linspace(0, 0.01, 99)
    fd = tempfile.TemporaryFile(suffix='csv')
    mlab.rec2csv(expected,fd)
    fd.seek(0)
    actual = mlab.csv2rec(fd)
    fd.close()
    assert np.allclose( expected['x'], actual['x'] )
    assert np.allclose( expected['y'], actual['y'] )
    assert np.allclose( expected['t'], actual['t'] )

def test_recarray_csv_roundtrip():
    expected = np.recarray((99,),
                          [('x',np.float),('y',np.float),('t',np.float)])
    expected['x'][0] = 1
    expected['y'][1] = 2
    expected['t'][2] = 3
    fd = tempfile.TemporaryFile(suffix='csv')
    mlab.rec2csv(expected,fd)
    fd.seek(0)
    actual = mlab.csv2rec(fd)
    fd.close()
    assert np.allclose( expected['x'], actual['x'] )
    assert np.allclose( expected['y'], actual['y'] )
    assert np.allclose( expected['t'], actual['t'] )

    def __call__(self, *args, **kwargs):
        """Load requested dataset, downloading it if needed or requested.

        For test purpose, instead of actually fetching the dataset, this
        function creates empty files and return their paths.
        """
        kwargs['mock'] = True
        files = original_fetch_files(*args, **kwargs)
        # Fill CSV files with given content if needed
        for f in files:
            basename = os.path.basename(f)
            if basename in self.csv_files:
                array = self.csv_files[basename]
                rec2csv(array, f)
        return files

def write_results_to_csv(results, directory):

    experiments, outcomes = results
#     deceased_pop = outcomes['relative market price']
#     time = outcomes[TIME]
    
    rec2csv(experiments, directory+'/experiments.csv', withheader=True)
    
    for key, value in outcomes.iteritems():
        np.savetxt(directory+'/{}.csv'.format(key), value, delimiter=',')
#     np.savetxt('./data/scarcity/relative_market_price.csv', deceased_pop, delimiter=',')
#     np.savetxt('./data/scarcity/time.csv', time, delimiter=',')
#     
    for entry in experiments.dtype.descr:
        print entry

def test_recarray_csv_roundtrip():
    expected = np.recarray((99,),
                          [('x',np.float),('y',np.float),('t',np.float)])
    # initialising all values: uninitialised memory sometimes produces floats
    # that do not round-trip to string and back.
    expected['x'] = np.linspace(0,1e-200,99)
    expected['y'] = np.linspace(0,1,99)
    expected['t'] = np.linspace(0,1e300,99)
    fd = tempfile.TemporaryFile(suffix='csv', mode="w+")
    mlab.rec2csv(expected,fd)
    fd.seek(0)
    actual = mlab.csv2rec(fd)
    fd.close()
    assert np.allclose( expected['x'], actual['x'] )
    assert np.allclose( expected['y'], actual['y'] )
    assert np.allclose( expected['t'], actual['t'] )

def main(basin_csv, basin_poly, storage_pts, stor_csv):
    basin_rec = mlab.csv2rec(basin_csv)

    ids = basin_rec["basinid"]
    d_ids = basin_rec["dwnbasinid"]

    ap.Identity_analysis(storage_pts, basin_poly, TMP_OUT, "NO_FID")
    out = ap.da.FeatureClassToNumPyArray(TMP_OUT,[STOR_FIELD,BASIN_ID_FIELD])

    stor = np.array([np.sum(out[STOR_FIELD][out[BASIN_ID_FIELD]==i]) for i in ids])

    fa_stor = fa.accumulate(ids,d_ids,f0,f,stor)

    outrec = np.rec.fromarrays((ids,stor,fa_stor),names=("basinid","stor","fa_stor"))
    
    mlab.rec2csv(outrec, stor_csv)

def test():
    """Test script"""
    import matplotlib.mlab as mlab
    import time
    import gen_merge

    BASINCSV = r"C:\Users\francis.gassert\Documents\ArcGIS\GISSync\global_maps\basins_15006.csv"
    BASINID = "basinid"
    DWNBASIN = "dwnbasinid"
    OUTCSV = r"C:\Users\francis.gassert\Documents\ArcGIS\GISSync\global_maps\bt_test.csv"
    runoffcsv = r"C:\Users\francis.gassert\Documents\ArcGIS\GISSync\global_maps\global-GLDAS-2.0_Noah-3.3_M.020-20121211-filled-20130821-RO.csv"
    
    basin_arr = mlab.csv2rec(BASINCSV)
    ids = basin_arr[BASINID]
    d_ids = basin_arr[DWNBASIN]
    r_arr = mlab.csv2rec(runoffcsv)
    r = r_arr["2010"]
    assert np.all(r_arr[BASINID]==ids)
    
    def f0( i, r ):
        return r[i]
    def f( i, idx, values, *args ):
        return np.sum(values[idx]) + f0(i, *args)
    
    
    time.clock()
    #id_dict = dict(zip(ids, upstream_ids(ids, d_ids)))
    #r2 = gen_merge.arrange_vector_by_ids(r, ids, np.arange(len(ids)+1))
    #out1 = np.array([np.sum(r2[id_dict[i]])+r2[i] for i in ids])
    #t1 = time.clock()

    out2 = accumulate(ids, d_ids, f0, f, r)
    t2 = time.clock()
    
    btcsv = r"C:\Users\francis.gassert\Documents\ArcGIS\GISSync\global_maps\global-GLDAS-2.0_Noah-3.3_M.020-20121211-filled-20130821-Bt.csv"
    bt_arr = mlab.csv2rec(btcsv)
    bt = bt_arr["2010"]

    #print ("time1: %s" % t1)
    print ("time2: %s" % t2)

    #print ("error1: %s " % (np.sum(out1-bt)/np.sum(bt)) )
    print ("error2: %s " % (np.sum(out2-bt)/np.sum(bt)) )
    
    outrec2 = np.rec.fromarrays((ids,out2),names=(BASINID,"2010"))
    mlab.rec2csv(outrec2,OUTCSV)