import water.fvec.Frame; //导入依赖的package包/类
private static void runKMeans() {
  File f = new File("/Users/apple/Downloads/whiskey.csv");
  NFSFileVec nfs = NFSFileVec.make(f);
  Frame frame = water.parser.ParseDataset.parse(Key.make(), nfs._key);
  KMeansModel.KMeansParameters params = new KMeansModel.KMeansParameters();
  params._train = frame._key;
  params._k = 5;
  params._max_iterations = 1000;
  KMeans job = new KMeans(params);
  KMeansModel kmm = job.trainModel().get();
  Frame output = kmm.score(frame);
  Vec[] vecs = output.vecs();
  System.out.print(Arrays.toString(vecs));
  for (Vec v : vecs) {
    System.out.println(v + ", " + v.length());
    for (int i = 0; i < v.length(); i++) {
      System.out.println(i + ": " + v.at(i));
    }
  }
  System.out.println();
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Ignore //PUBDEV-1001
@Test public void testCzechboard() throws Throwable {
  basicDLTest_Classification(
      "./smalldata/gbm_test/czechboard_300x300.csv", "czechboard_300x300.hex",
      new PrepData() {
        @Override
        int prep(Frame fr) {
          Vec resp = fr.remove("C2");
          fr.add("C2", resp.toCategoricalVec());
          resp.remove();
          return fr.find("C3");
        }
      },
      1,
      ard(ard(7, 44993),
          ard(2, 44998)),
      s("0", "1"),
      DeepLearningParameters.Activation.Tanh);
}
 

import water.fvec.Frame; //导入依赖的package包/类
static Vec unifyFrame(DeepLearningParameters drf, Frame fr, PrepData prep, boolean classification) {
  int idx = prep.prep(fr);
  if( idx < 0 ) { idx = ~idx; }
  String rname = fr._names[idx];
  drf._response_column = fr.names()[idx];

  Vec resp = fr.vecs()[idx];
  Vec ret = null;
  if (classification) {
    ret = fr.remove(idx);
    fr.add(rname,resp.toCategoricalVec());
  } else {
    fr.remove(idx);
    fr.add(rname,resp);
  }
  return ret;
}
 

import water.fvec.Frame; //导入依赖的package包/类
/**
 * Helper to compute the initial value for Bernoulli for offset != 0
 * @return
 */
private double getInitialValueBernoulliOffset(Frame train) {
  Log.info("Running Newton-Raphson iteration to find the initial value since offsets are specified.");
  double delta;
  int count = 0;
  double tol = 1e-4;

  //From R GBM vignette:
  //For speed, gbm() does only one step of the Newton-Raphson algorithm
  //rather than iterating to convergence. No appreciable loss of accuracy
  //since the next boosting iteration will simply correct for the prior iterations
  //inadequacy.
  int N = 1; //one step is enough - same as R

  double init = 0; //start with initial value of 0 for convergence
  do {
    double newInit = new NewtonRaphson(init).doAll(train).value();
    delta = Math.abs(init - newInit);
    init = newInit;
    Log.info("Iteration " + ++count + ": initial value: " + init);
  } while (count < N && delta >= tol);
  if (delta > tol) Log.warn("Not fully converged.");
  Log.info("Newton-Raphson iteration ran for " + count + " iteration(s). Final residual: " + delta);
  return init;
}
 

import water.fvec.Frame; //导入依赖的package包/类
/** Simple testing scenario, splitting frame in the middle and comparing the values */
static void testScenario(Frame f, String[] expValues) {
  double[] ratios = ard(0.5, 0.5);
  Key[] keys = aro(Key.make("test.hex"), Key.make("train.hex"));
  Frame[] splits = null;
  try {
    splits = ShuffleSplitFrame.shuffleSplitFrame(f, keys, ratios, 42);
    Assert.assertEquals("Expecting 2 splits", 2, splits.length);
    // Collect values from both splits
    String[] values = append(
            collectS(splits[0].vec(0)),
            collectS(splits[1].vec(0)));
    // Sort values, but first replace all nulls by unique value
    Arrays.sort(replaceNulls(expValues));
    Arrays.sort(replaceNulls(values));
    Assert.assertArrayEquals("Values should match", expValues, values);
  } finally {
    f.delete();
    if (splits!=null) for(Frame s: splits) s.delete();
  }
}
 

import water.fvec.Frame; //导入依赖的package包/类
ScoreBuildOneTree(SharedTree st, int k, int nbins, int nbins_cats, DTree tree, int leafs[], DHistogram hcs[][][], Frame fr2, boolean subset, boolean build_tree_one_node, float[] improvPerVar, Distribution.Family family) {
  _st   = st;
  _k    = k;
  _nbins= nbins;
  _nbins_cats= nbins_cats;
  _tree = tree;
  _leafs= leafs;
  _hcs  = hcs;
  _fr2  = fr2;
  _subset = subset;
  _build_tree_one_node = build_tree_one_node;
  _improvPerVar = improvPerVar;
  _family = family;
  // Raise the priority, so that if a thread blocks here, we are guaranteed
  // the task completes (perhaps using a higher-priority thread from the
  // upper thread pools).  This prevents thread deadlock.
  _priority = nextThrPriority();
}
 

import water.fvec.Frame; //导入依赖的package包/类
/** Score an already adapted frame.  Returns a MetricBuilder that can be used to make a model metrics.
 * @param adaptFrm Already adapted frame
 * @return MetricBuilder
 */
protected ModelMetrics.MetricBuilder scoreMetrics(Frame adaptFrm) {
  final boolean computeMetrics = (!isSupervised() || adaptFrm.find(_output.responseName()) != -1);
  // Build up the names & domains.
  final int nc = _output.nclasses();
  final int ncols = nc==1?1:nc+1; // Regression has 1 predict col; classification also has class distribution
  String[] names = new String[ncols];
  String[][] domains = new String[ncols][];
  names[0] = "predict";
  for(int i = 1; i < names.length; ++i) {
    names[i] = _output.classNames()[i - 1];
    // turn integer class labels such as 0, 1, etc. into p0, p1, etc.
    try {
      Integer.valueOf(names[i]);
      names[i] = "p" + names[i];
    } catch (Throwable t) {
      // do nothing, non-integer names are fine already
    }
  }
  domains[0] = nc==1 ? null : !computeMetrics ? _output._domains[_output._domains.length-1] : adaptFrm.lastVec().domain();
  // Score the dataset, building the class distribution & predictions
  BigScore bs = new BigScore(domains[0],ncols,adaptFrm.means(),_output.hasWeights() && adaptFrm.find(_output.weightsName()) >= 0,computeMetrics, false /*no preds*/).doAll(adaptFrm);
  return bs._mb;
}
 

import water.fvec.Frame; //导入依赖的package包/类
@SuppressWarnings("unused") // called through reflection by RequestServer
public DownloadDataV3 fetch(int version, DownloadDataV3 server) {

  if (DKV.get(server.frame_id.key()) == null) throw new H2OKeyNotFoundArgumentException("key", server.frame_id.key());
  Frame value = server.frame_id.key().get();
  InputStream is = value.toCSV(true, server.hex_string);
  java.util.Scanner scanner = new java.util.Scanner(is).useDelimiter("\\A");
  server.csv = (scanner.hasNext() ? scanner.next() : "");

  // Clean up Key name back to something resembling a file system name.  Hope
  // the user's browser actually asks for what to do with the suggested
  // filename.  Without this code, my FireFox would claim something silly
  // like "no helper app installed", then fail the download.
  String s = server.frame_id.toString();
  int x = s.length()-1;
  boolean dot=false;
  for( ; x >= 0; x-- )
    if( !Character.isLetterOrDigit(s.charAt(x)) && s.charAt(x)!='_' )
      if( s.charAt(x)=='.' && !dot ) dot=true;
      else break;
  String suggested_fname = s.substring(x+1).replace(".hex", ".csv");
  if( !suggested_fname.endsWith(".csv") )
    suggested_fname = suggested_fname+".csv";
  server.filename = suggested_fname;
  return server;
}
 

import water.fvec.Frame; //导入依赖的package包/类
public KeyV3(Key key) {
  this();
  if (null != key) {
    Class clz = getKeyedClass();
    Value v = DKV.get(key);

    if (null != v) {
      // Type checking of value from DKV
      if (Job.class.isAssignableFrom(clz) && !v.isJob())
        throw new H2OIllegalArgumentException("For Key: " + key + " expected a value of type Job; found a: " + v.theFreezableClass(), "For Key: " + key + " expected a value of type Job; found a: " + v.theFreezableClass() + " (" + clz + ")");
      else if (Frame.class.isAssignableFrom(clz) && !v.isFrame() && !v.isVec())
      // NOTE: we currently allow Vecs to be fetched via the /Frames endpoint, so this constraint is relaxed accordingly.  Note this means that if the user gets hold of a (hidden) Vec key and passes it to some other endpoint they will get an ugly error instead of an H2OIllegalArgumentException.
        throw new H2OIllegalArgumentException("For Key: " + key + " expected a value of type Frame; found a: " + v.theFreezableClass(), "For Key: " + key + " expected a value of type Frame; found a: " + v.theFreezableClass() + " (" + clz + ")");
      else if (Model.class.isAssignableFrom(clz) && !v.isModel())
        throw new H2OIllegalArgumentException("For Key: " + key + " expected a value of type Model; found a: " + v.theFreezableClass(), "For Key: " + key + " expected a value of type Model; found a: " + v.theFreezableClass() + " (" + clz + ")");
      else if (Vec.class.isAssignableFrom(clz) && !v.isVec())
        throw new H2OIllegalArgumentException("For Key: " + key + " expected a value of type Vec; found a: " + v.theFreezableClass(), "For Key: " + key + " expected a value of type Vec; found a: " + v.theFreezableClass() + " (" + clz + ")");
    }

    this.fillFromImpl(key);
  }
}
 

import water.fvec.Frame; //导入依赖的package包/类
/**
 * Transpose the Frame as if it was a matrix (i.e. rows become coumns).
 * Must be all numeric, currently will fail if there are too many rows ( >= ~.5M).
 * Result will be put into a new Vectro Group and will be balanced so that each vec will have
 * (4*num cpus in the cluster) chunks.
 *
 * @param src
 * @return
 */
public static Frame transpose(Frame src){
  if(src.numRows() != (int)src.numRows())
    throw H2O.unimpl();
  int nchunks = Math.max(1,src.numCols()/10000);
  long [] espc = new long[nchunks+1];
  int rpc = (src.numCols() / nchunks);
  int rem = (src.numCols() % nchunks);
  Arrays.fill(espc, rpc);
  for (int i = 0; i < rem; ++i) ++espc[i];
  long sum = 0;
  for (int i = 0; i < espc.length; ++i) {
    long s = espc[i];
    espc[i] = sum;
    sum += s;
  }
  Key key = Vec.newKey();
  int rowLayout = Vec.ESPC.rowLayout(key,espc);
  return transpose(src, new Frame(new Vec(key,rowLayout).makeZeros((int)src.numRows())));
}
 

import water.fvec.Frame; //导入依赖的package包/类
/**
 * Helper to create the DataInfo object from training/validation frames and the DL parameters
 * @param train Training frame
 * @param valid Validation frame
 * @param parms Model parameters
 * @return
 */
static DataInfo makeDataInfo(Frame train, Frame valid, DeepLearningParameters parms) {
  double x = 0.782347234;
  boolean identityLink = new Distribution(parms._distribution, parms._tweedie_power).link(x) == x;
  return new DataInfo(
          Key.make(), //dest key
          train,
          valid,
          parms._autoencoder ? 0 : 1, //nResponses
          parms._autoencoder || parms._use_all_factor_levels, //use all FactorLevels for auto-encoder
          parms._autoencoder ? DataInfo.TransformType.NORMALIZE : parms._sparse ? DataInfo.TransformType.DESCALE : DataInfo.TransformType.STANDARDIZE, //transform predictors
          train.lastVec().isCategorical() ? DataInfo.TransformType.NONE : identityLink ? DataInfo.TransformType.STANDARDIZE : DataInfo.TransformType.NONE, //transform response for regression with identity link
          parms._missing_values_handling == DeepLearningParameters.MissingValuesHandling.Skip, //whether to skip missing
          false, // do not replace NAs in numeric cols with mean
          true,  // always add a bucket for missing values
          parms._weights_column != null, // observation weights
          parms._offset_column != null,
          parms._fold_column != null
    );
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Test public void testCreditProstateMaxout() throws Throwable {
  basicDLTest_Classification(
      "./smalldata/logreg/prostate.csv", "prostateMaxout.hex",
      new PrepData() {
        @Override
        int prep(Frame fr) {
          fr.remove("ID").remove();
          return fr.find("CAPSULE");
        }
      },
      100,
      ard(ard(189, 38),
          ard(30, 123)),
      s("0", "1"),
      DeepLearningParameters.Activation.Maxout);
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Test public void testModelAdaptConvert() {
  AModel.AParms p = new AModel.AParms();
  AModel.AOutput o = new AModel.AOutput();

  Frame trn = new Frame();
  trn.add("dog",vec(new String[]{"A","B"},0,1,0,1));
  o._names = trn.names();
  o._domains = trn.domains();
  trn.remove();
  AModel am = new AModel(Key.make(),p,o);
  
  Frame tst = new Frame();
  tst.add("dog",vec(2, 3, 2, 3));
  Frame adapt = new Frame(tst);
  boolean saw_iae = false;
  try { am.adaptTestForTrain(adapt, true, true); }
  catch( IllegalArgumentException iae ) { saw_iae = true; }
  Assert.assertTrue(saw_iae);

  Model.cleanup_adapt( adapt, tst );
  tst.remove();
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Override public S fillFromImpl(ModelMetrics modelMetrics) {
  // If we're copying in a Model we need a ModelSchema of the right class to fill into.
  Model m = modelMetrics.model();
  if( m != null ) {
    this.model = new ModelKeyV3(m._key);
    this.model_category = m._output.getModelCategory();
    this.model_checksum = m.checksum();
  }

  // If we're copying in a Frame we need a Frame Schema of the right class to fill into.
  Frame f = modelMetrics.frame();
  if (null != f) { //true == f.getClass().getSuperclass().getGenericSuperclass() instanceof ParameterizedType
    this.frame = new FrameKeyV3(f._key);
    this.frame_checksum = f.checksum();
  }

  PojoUtils.copyProperties(this, modelMetrics, PojoUtils.FieldNaming.ORIGIN_HAS_UNDERSCORES,
          new String[]{"model", "model_category", "model_checksum", "frame", "frame_checksum"});


  return (S) this;
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Override public ModelMetricsListSchemaV3 fillFromImpl(ModelMetricsList mml) {
  // TODO: this is failing in PojoUtils with an IllegalAccessException.  Why?  Different class loaders?
  // PojoUtils.copyProperties(this, m, PojoUtils.FieldNaming.CONSISTENT);

  // Shouldn't need to do this manually. . .
  this.model = (mml._model == null ? null : new KeyV3.ModelKeyV3(mml._model._key));
  this.frame = (mml._frame == null ? null : new KeyV3.FrameKeyV3(mml._frame._key));
  this.predictions_frame = (mml._predictions_name == null ? null : new KeyV3.FrameKeyV3(Key.<Frame>make(mml._predictions_name)));
  this.reconstruction_error = mml._reconstruction_error;
  this.reconstruction_error_per_feature = mml._reconstruction_error_per_feature;
  this.deep_features_hidden_layer = mml._deep_features_hidden_layer;

  if (null != mml._model_metrics) {
    this.model_metrics = new ModelMetricsBase[mml._model_metrics.length];
    for( int i=0; i<model_metrics.length; i++ ) {
      ModelMetrics mm = mml._model_metrics[i];
      this.model_metrics[i] = (ModelMetricsBase) Schema.schema(3, mm.getClass()).fillFromImpl(mm);
    }
  } else {
    this.model_metrics = new ModelMetricsBase[0];
  }
  return this;
}
 

import water.fvec.Frame; //导入依赖的package包/类
private Frame checkTree(String tree, boolean expectThrow) {
  Frame fr = parse_test_file(Key.make("a.hex"),"smalldata/iris/iris_wheader.csv");
  fr.remove(4).remove();
  try {
    Val val = Exec.exec(tree);
    Assert.assertFalse(expectThrow);
    System.out.println(val.toString());
    if( val instanceof ValFrame )
      return ((ValFrame)val)._fr;
    throw new IllegalArgumentException("exepcted a frame return");
  } catch( IllegalArgumentException iae ) {
    if( !expectThrow ) throw iae; // If not expecting a throw, then throw which fails the junit
    fr.delete();                  // If expecting, then cleanup
    return null;
  }
}
 

import water.fvec.Frame; //导入依赖的package包/类
private GBMModel prepareGBMModel(String dataset, String[] ignoredColumns, String response, boolean classification, int ntrees) {
  Frame f = parse_test_file(dataset);
  try {
    if (classification && !f.vec(response).isCategorical()) {
      f.replace(f.find(response), f.vec(response).toCategoricalVec()).remove();
      DKV.put(f._key, f);
    }
    GBMModel.GBMParameters gbmParams = new GBMModel.GBMParameters();
    gbmParams._train = f._key;
    gbmParams._ignored_columns = ignoredColumns;
    gbmParams._response_column = response;
    gbmParams._ntrees = ntrees;
    gbmParams._score_each_iteration = true;
    return new GBM(gbmParams).trainModel().get();
  } finally {
    if (f!=null) f.delete();
  }
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Test public void testIris() {
  QuantileModel kmm = null;
  Frame fr = null;
  try {
    long start = System.currentTimeMillis();
    System.out.println("Start Parse");
    fr = parse_test_file("smalldata/iris/iris_wheader.csv");
    //fr = parse_test_file("../../datasets/UCI/UCI-large/covtype/covtype.data");
    //fr = parse_test_file("../../datasets/billion_rows.csv.gz");
    System.out.println("Done Parse: "+(System.currentTimeMillis()-start));

    QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
    parms._train = fr._key;
    //parms._max_iterations = 10;

    Job<QuantileModel> job = new Quantile(parms).trainModel();
    kmm = job.get();
    job.remove();

  } finally {
    if( fr  != null ) fr .remove();
    if( kmm != null ) kmm.delete();
  }
}
 

import water.fvec.Frame; //导入依赖的package包/类
@SuppressWarnings("unused") // called through reflection by RequestServer
// TODO: return list of FrameSummaryV3 that has histograms et al.
public FramesV3 summary(int version, FramesV3 s) {
  Frame frame = getFromDKV("key", s.frame_id.key()); // safe

  if (null != frame) {
    Futures fs = new Futures();
    Vec[] vecArr = frame.vecs();
    for (Vec v : vecArr) {
      if (! v.isString()) {
        v.startRollupStats(fs, Vec.DO_HISTOGRAMS);
      }
    }
    fs.blockForPending();
  }

  return doFetch(version, s, FrameV3.ColV3.FORCE_SUMMARY);
}
 

import water.fvec.Frame; //导入依赖的package包/类
@Test public void test50pct() {
  QuantileModel kmm = null;
  Frame fr = null;
  try {
    double[][] d = new double[][]{{1.56386606237}, {0.812834256224}, {3.68417563302}, {3.12702210880}, {5.51277746586}};
    fr = ArrayUtils.frame(d);
    QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
    parms._train = fr._key;
    Job<QuantileModel> job = new Quantile(parms).trainModel();
    kmm = job.get();
    job.remove();
    Assert.assertTrue(kmm._output._quantiles[0][5]==d[3][0]);

  } finally {
    if( fr  != null ) fr .remove();
    if( kmm != null ) kmm.delete();
  }
}