Scala PCA类代码示例

OStack程序员社区-中国程序员成长平台 › 门户 › 编程› Scala›Scala教程

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Scala中org.apache.spark.ml.feature.PCA类的典型用法代码示例。如果您正苦于以下问题：Scala PCA类的具体用法？Scala PCA怎么用？Scala PCA使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了PCA类的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Scala代码示例。

示例1: PCASampleDemo

//设置package包名称以及导入依赖的类
package com.chapter11.SparkMachineLearning

import org.apache.spark.ml.feature.PCA
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.sql.SparkSession

object PCASampleDemo {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .master("local[4]")
      .appName("PCAExample")
      .getOrCreate()

    val data = Array(
       Vectors.dense(3.5, 2.0, 5.0, 6.3, 5.60, 2.4),
       Vectors.dense(4.40, 0.10, 3.0, 9.0, 7.0, 8.75),
       Vectors.dense(3.20, 2.40, 0.0, 6.0, 7.4, 3.34)
    )
    val df = spark.createDataFrame(data.map(Tuple1.apply)).toDF("features")
    df.show(false)

    val pca = new PCA()
      .setInputCol("features")
      .setOutputCol("pcaFeatures")
      .setK(4)
      .fit(df)

    val result = pca.transform(df).select("pcaFeatures")
    result.show(false)

    spark.stop()
  }
}

开发者ID:PacktPublishing，项目名称:Scala-and-Spark-for-Big-Data-Analytics，代码行数:35，代码来源:PCAExample.scala

示例2: Pca

//设置package包名称以及导入依赖的类
package com.github.dongjinleekr.spark.example

import com.github.dongjinleekr.spark.dataset.Iris
import com.github.dongjinleekr.spark.dataset.Iris._
import org.apache.spark.ml.feature.{PCA, VectorAssembler}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.SparkSession


object Pca {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("PCA Example")
      .getOrCreate()

    // Read the file
    val raw = spark.read
      .schema(Iris.schema)
      .option("header", true)
      .csv("hdfs:///datasets/iris/data.csv")

    // Normalize:
    // 1. Combine the features into vector.
    // 2. Convert enumerating value into Int type.
    val assembler = new VectorAssembler()
      .setInputCols(Iris.schema.fields.map(_.name).slice(1, 5))
      .setOutputCol("features")

    def speciesToInt: (String => Int) = { s: String => Species.toInt(s) }

    val newSpecies = udf(speciesToInt).apply(col("species"))
    val df = assembler.transform(raw)
      .withColumn("species", newSpecies)
      .select("id", "features", "species")

    // PCA (2)
    val pca = new PCA()
      .setInputCol("features")
      .setOutputCol("pcaFeatures")
      .setK(2)
      .fit(df)

    val result = pca.transform(df).select("pcaFeatures")
    result.show(false)
  }
}

开发者ID:dongjinleekr，项目名称:spark-dataset，代码行数:48，代码来源:Pca.scala

示例3: PCAJob

//设置package包名称以及导入依赖的类
import io.hydrosphere.mist.api._
import io.hydrosphere.mist.api.ml._
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.feature.PCA
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.sql.SparkSession
//TODO: why model return vector from mllib??
import org.apache.spark.mllib.linalg.{Vector => OldVector}

object PCAJob extends MLMistJob {
  def session: SparkSession = SparkSession
    .builder()
    .appName(context.appName)
    .config(context.getConf)
    .getOrCreate()

  def train(savePath: String): Map[String, Any] = {
    val data = Array(
      Vectors.sparse(5, Seq((1, 1.0), (3, 7.0))),
      Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0),
      Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0)
    )
    val df = session.createDataFrame(data.map(Tuple1.apply)).toDF("features")

    val pca = new PCA()
      .setInputCol("features")
      .setOutputCol("pcaFeatures")
      .setK(3)

    val pipeline = new Pipeline().setStages(Array(pca))

    val model = pipeline.fit(df)

    model.write.overwrite().save(savePath)
    Map.empty[String, Any]
  }

  def serve(modelPath: String, features: List[Array[Double]]): Map[String, Any] = {
    import LocalPipelineModel._

    val pipeline = PipelineLoader.load(modelPath)
    val data = LocalData(LocalDataColumn("features", features))

    val result = pipeline.transform(data).toMapList.map(rowMap => {
      rowMap + ("pcaFeatures" -> rowMap("pcaFeatures").asInstanceOf[OldVector].toArray)
    })
    Map("result" -> result)
  }
}

开发者ID:Hydrospheredata，项目名称:mist，代码行数:50，代码来源:PCAJob.scala

示例4: TestPcaExample

//设置package包名称以及导入依赖的类
package com.burness.algorithm.feature

import org.apache.spark.SparkConf
import org.apache.spark.ml.feature.PCA
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.sql.SparkSession


object  TestPcaExample{
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]")
    val spark = SparkSession
      .builder
      .appName("PCAExample")
      .config(sparkConf)
      .getOrCreate()

    // $example on$
    val data = Array(
      Vectors.sparse(5, Seq((1, 1.0), (3, 7.0))),
      Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0),
      Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0)
    )
    val df = spark.createDataFrame(data.map(Tuple1.apply)).toDF("features")
    val pca = new PCA()
      .setInputCol("features")
      .setOutputCol("pcaFeatures")
      .setK(3)
      .fit(df)
    val pcaDF = pca.transform(df)
    val result = pcaDF.select("pcaFeatures")
    result.rdd.foreach{
      case s =>
        println(s)
    }
    // $example off$

    spark.stop()
  }

}

开发者ID:spark-mler，项目名称:algorithmEngine，代码行数:42，代码来源:test_pca_example.scala

注：本文中的org.apache.spark.ml.feature.PCA类示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。