//设置package包名称以及导入依赖的类
package ml.sparkling.graph.loaders.csv

import org.apache.spark.sql.DataFrameReader


object DataFrameReaderConfigurator {

  implicit class addAbilityToConfigureDataFrameReader(reader:DataFrameReader){
    def applyConfiguration(cSVLoaderConfig: CsvLoaderConfig):DataFrameReader={
      reader.option("header",cSVLoaderConfig.header.toString)
      reader.option("delimiter",cSVLoaderConfig.delimiter)
      reader.option("quote",cSVLoaderConfig.quote)
     cSVLoaderConfig.schema match{
       case None => reader.option("inferSchema",cSVLoaderConfig.inferSchema.toString)
       case Some(schema) => reader.schema(schema)
      }
      reader
    }
  }

} 

//设置package包名称以及导入依赖的类
package test.yumi.pipeline

import com.typesafe.config.Config
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrameReader, SparkSession}
import org.apache.spark.sql.types.StructType
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import yumi.Job
import yumi.metastore.Metastore
import yumi.pipeline._

trait MockSessionSpec extends BaseSpec {

  trait MockSessionScope extends BaseMockScope {

    implicit val yumiContext = mock[YumiContext]
    val pipelineFactory = mock[PipelineFactory]
    val activityLoader = mock[ActivityLoader]
    val contextFactory = mock[YumiContextFactory]
    val sparkSession = mock[SparkSession]
    val sparkContext = mock[SparkContext]
    val dataFrameReader = mock[DataFrameReader]
    val dataFrameWriter = mock[DataFrameWriter]
    val emptyParameters = new Parameters

    when(sparkSession.sparkContext).thenReturn(sparkContext)

    when(yumiContext.sparkSession).thenReturn(sparkSession)
    when(yumiContext.sparkContext).thenReturn(sparkContext)
    when(yumiContext.dataFrameWriter).thenReturn(dataFrameWriter)

    when(sparkSession.read).thenReturn(dataFrameReader)

    when(dataFrameReader.format(any[String])).thenReturn(dataFrameReader)
    when(dataFrameReader.schema(any[StructType])).thenReturn(dataFrameReader)
    when(dataFrameReader.option(any[String], any[String])).thenReturn(dataFrameReader)
    when(dataFrameReader.options(any[Map[String, String]])).thenReturn(dataFrameReader)
  }
} 

//设置package包名称以及导入依赖的类
package com.github.traviscrawford.spark

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.DataFrameReader

package object dynamodb {

  implicit class DynamoDBDataFrameReader(reader: DataFrameReader) {

    
    def dynamodb(region: String, table: String): DataFrame =
      reader
        .format("com.github.traviscrawford.spark.dynamodb")
        .option("region", region)
        .option("table", table)
        .load
  }
}