//设置package包名称以及导入依赖的类
package util

import java.io.ByteArrayInputStream
import java.nio.charset.{Charset, CodingErrorAction}

import text.StringOption

import scala.collection.mutable.ListBuffer
import scala.io.{Codec, Source}
import scala.sys.process.ProcessBuilder


object ProcessBuilderUtils {
  implicit def processToProcessUtils(repr: ProcessBuilder): ProcessBuilderUtils = {
    new ProcessBuilderUtils(repr)
  }
}

class ProcessBuilderUtils(repr: ProcessBuilder) {
  def lineStream(encoding: Charset,
                 onMalformedInput: CodingErrorAction,
                 onUnmappableCharacter: CodingErrorAction,
                 replacementOpt: StringOption): Iterator[String] = {
    val lines: Iterator[String] = repr.lineStream_!.iterator
    val byteBuffer = ListBuffer.empty[Byte]
    while (lines.hasNext) {
      val line: String = lines.next.trim concat "\n"
      byteBuffer ++= line.getBytes
    }
    implicit val codec = Codec(encoding).
      onMalformedInput(onMalformedInput).
      onUnmappableCharacter(onUnmappableCharacter)
    if (replacementOpt.nonEmpty) {
      codec.decodingReplaceWith(replacementOpt.get)
    }
    Source.fromInputStream(new ByteArrayInputStream(byteBuffer.toArray)).getLines
  }
} 

//设置package包名称以及导入依赖的类
package codacy.dockerApi.utils

import java.io._
import java.nio.charset.CodingErrorAction

import scala.collection.mutable
import scala.io.{Codec, Source}
import scala.language.postfixOps
import scala.sys.process._
import scala.util.{Failure, Success, Try}

case class CommandResult(exitCode: Int, stdout: List[String], stderr: List[String])

object CommandRunner {

  def exec(cmd: List[String], dir: Option[File] = None): Either[Throwable, CommandResult] = {
    val stdout = mutable.Buffer[String]()
    val stderr = mutable.Buffer[String]()

    val pio = new ProcessIO(_.close(), readStream(stdout), readStream(stderr))

    Try(Process(cmd, dir).run(pio)) match {
      case Success(process) =>
        Try(process.exitValue()) match {
          case Success(exitValue) =>
            Right(CommandResult(exitValue, stdout.toList, stderr.toList))

          case Failure(e) =>
            process.destroy()
            Left(e)
        }

      case Failure(e) =>
        Left(e)
    }
  }

  private def readStream(buffer: mutable.Buffer[String])(stream: InputStream) = {
    implicit val codec = Codec("UTF-8")
    codec.onMalformedInput(CodingErrorAction.IGNORE)
    codec.onUnmappableCharacter(CodingErrorAction.IGNORE)

    Source.fromInputStream(stream)
      .getLines()
      .foreach { line => buffer += line }
    stream.close()
  }

} 

//设置package包名称以及导入依赖的类
package controllers

import authes.AuthConfigImpl
import authes.Role.Administrator
import com.github.tototoshi.play2.json4s.Json4s
import com.google.inject.Inject
import jp.t2v.lab.play2.auth.AuthElement
import net.liftweb.util.Html5
import org.json4s.{DefaultFormats, Extraction}
import play.api.mvc.{Action, Controller}
import scrape.model.{StationPage, TrainPage}

import scala.io.Codec
import scala.xml._

class Scraper @Inject() (json4s: Json4s) extends Controller with AuthElement with AuthConfigImpl {
  import Responses._
  import Scraper._
  import json4s._

  implicit val formats = DefaultFormats

  def station(lineId: String, pageName: String) = Action { implicit req =>
    val url = s"${Host}/newdata/ekijikoku/${lineId}/${pageName}.htm"
    val xml = loadXML(url)
    StationPage.fromXml(xml) match {
      case Left(str) => notFound(str)
      case Right(station) =>
        val trains = station.trains.filterNot(_.add.contains("?"))
        Ok(Extraction.decompose(trains.map(_.replaceAbbr(station.abbr))))
    }
  }

  def timeTable(lineId: String, pageName: String) = StackAction(AuthorityKey -> Administrator) { implicit req =>
    ???
  }

  def train(lineId: String, trainId: String) = StackAction(AuthorityKey -> Administrator) { implicit req =>
    val address = s"/newdata/detail/${lineId}/${trainId}.htm"
    TrainPage.fromXML(loadXML(s"${Host}${address}"), address) match {
      case Left(str) => notFound(str)
      case Right(train) => Ok(Extraction.decompose(train))
    }
  }

  private[this] def loadXML(url: String): NodeSeq = {
    val html = scala.io.Source.fromURL(url)(Codec("Shift_JIS")).mkString
    Html5.parse(html) openOr NodeSeq.Empty
  }
}

object Scraper {
  val Host = "http://www.ekikara.jp"
} 

//设置package包名称以及导入依赖的类
package io.rosenberg.elff

import scala.io.Codec
import java.nio.charset.CodingErrorAction


	def getLogItems(lines: Iterator[String]): Iterator[collection.mutable.Map[String, String]] = {
		// this is to avoid encoding errors that frequently happen in large log files
		if (_fixEncodingError) {
			implicit val codec = Codec("UTF-8")
			codec.onMalformedInput(CodingErrorAction.REPLACE)
			codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
		}
		return new LogItemsIterator(lines, fields)
	}
	
} 

//设置package包名称以及导入依赖的类
package com.codacy.dotnet.helpers

import java.io.InputStream
import java.nio.charset.{CodingErrorAction, StandardCharsets}
import java.nio.file.{Files, Path, StandardOpenOption}

import scala.io.Codec
import scala.util.{Failure, Try}

object ResourceHelper {

  implicit val codec = Codec("UTF-8")
  codec.onMalformedInput(CodingErrorAction.REPLACE)
  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)

  def getResourceStream(path: String): Try[InputStream] = {
    Option(getClass.getClassLoader.getResource(path)).map { url =>
      Try(url.openStream())
    }.getOrElse {
      Failure(new Exception("The path provided was not found"))
    }
  }

  def writeFile(path: Path, content: String): Try[Path] = {
    Try(Files.write(
      path,
      content.getBytes(StandardCharsets.UTF_8),
      StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE
    ))
  }

} 

//设置package包名称以及导入依赖的类
import java.io.BufferedOutputStream
import scala.io.Source
import java.io.File
import java.io.FileOutputStream
import scala.io.Codec
package object utils {

	
	implicit class RichFile(file: File) {
		def read() = Source.fromFile(file)(Codec.UTF8).mkString

		def write(data: String) {
			val fos = new BufferedOutputStream(new FileOutputStream(file))
			try {
				fos.write(data.getBytes("UTF-8"))
			} finally {
				fos.close
			}
		}
	}
} 

//设置package包名称以及导入依赖的类
package com.github.gigurra.io

import java.nio.charset.{Charset, StandardCharsets}

import scala.io.Codec


object LoadString {

  def from(location: String,
           lookForLocalResource: Boolean = true,
           lookForGlobalResource: Boolean = true,
           lookForFile: Boolean = true,
           charset: Charset = StandardCharsets.UTF_8): Option[String] = {

    val streamOpt = LoadInputStream.from(
      location = location,
      lookForLocalResource = lookForLocalResource,
      lookForGlobalResource = lookForGlobalResource,
      lookForFile = lookForFile
    )

    val out = streamOpt.map(stream => scala.io.Source.fromInputStream(stream)(Codec(charset)).mkString)
    streamOpt.foreach(_.close())
    out
  }
} 

//设置package包名称以及导入依赖的类
package com.ntsdev.service

import java.util.Collections

import org.neo4j.ogm.session.Session
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.annotation.Profile
import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.{Propagation, Transactional}

import scala.collection.JavaConversions._
import scala.io.{Codec, Source}

@Service
@Profile(Array("!cloud"))
@Transactional(propagation = Propagation.REQUIRED)
class TestDataService {
  private val log = LoggerFactory.getLogger(getClass)
  private var loaded = false

  @Autowired
  val session: Session = null

  def loadTestData() = {
    session.purgeDatabase()
    log.info ("Loading test data...")

    val emptyMap = mapAsJavaMap[String, AnyRef](Collections.emptyMap[String, AnyRef]())
    session.query(loadDataFromFile("testdata.cql"), emptyMap)

    log.info("Test data loaded.")
    loaded = true
  }

  private def loadDataFromFile(fileName: String): String = {
    Source.fromURL(getClass.getResource("/" + fileName))(Codec.UTF8).mkString
  }

} 

//设置package包名称以及导入依赖的类
import scala.io.Source
import java.nio.charset.CodingErrorAction
import scala.io.Codec

object Scraper1 {

  def main(args: Array[String]): Unit = {
    implicit val codec = Codec("UTF-8")
    codec.onMalformedInput(CodingErrorAction.IGNORE)
    codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
    val url = "http://blog.jungbin.kim"
    val html = Source.fromURL(url)
    val s = html.getLines.mkString("\n")
    println(s)
  }

} 

//设置package包名称以及导入依赖的类
package indexer

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import java.nio.charset.CodingErrorAction
import scala.io.Codec
import scala.io.Source

object Utilities {
  implicit val codec = Codec("UTF-8")
  codec.onMalformedInput(CodingErrorAction.REPLACE)
  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)

  val mapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  val stopWordsFile = "src/main/resources/stopwords"
  lazy val stopWords = Source.fromFile(stopWordsFile).getLines.map(_.replaceAll("'", "")).toSet

  def tokenize(source: Source): Iterator[(String, Int)] = source.getLines
    .map(_.replaceAll("""(?<=\w)[',](?=\w)""", ""))
    .map(_.replaceAll("""(?<=\d)[.](?=\d)""", "__TOKENZ__DECML__"))
    .flatMap(_.split("""\W+"""))
    .filterNot(_ == "")
    .map(_.replaceAll("__TOKENZ__DECML__", "."))
    .map(_.toLowerCase)
    .zipWithIndex
    .filterNot { case (token, index) => !token.matches(""".*[a-z].*""") && token.size > 4 }
    .filterNot { case (token, index) => token.size < 3 || token.size > 20 }
    .filterNot { case (token, index) => stopWords(token) }

  def nGrams(n: Integer, source: Source): Iterator[(String, Int)] = tokenize(source)
    .sliding(n)
    .filter { case item => item.last._2 - item.head._2 == n - 1 }
    .filter(_.length == n)
    .map(_.unzip)
    .map { case (tokens, pos :: _) => (tokens.mkString(" "), pos)  }

  def frequencies(tokens: Iterator[(String, Int)]): Iterator[(String, Int)] = tokens
    .foldLeft(Map[String, Int]()) { case (acc, (token, _)) =>
      acc + (token -> (acc.getOrElse(token, 0) + 1))
    }
    .toIterator

  def time[R](block: => R): R = {
    val t0 = System.nanoTime()
    val result = block    // call-by-name
    val t1 = System.nanoTime()
    println("Elapsed time: " + (t1 - t0) + "ns")
    result
  }
} 

//设置package包名称以及导入依赖的类
package util.process

import java.io.ByteArrayInputStream
import java.nio.charset.{Charset, CodingErrorAction}

import text.StringOption

import scala.io.{Codec, Source}
import scala.sys.process.ProcessBuilder


class ProcessBuilderUtils(repr: ProcessBuilder) {
  def lineStream(encoding: Charset,
                 onMalformedInput: CodingErrorAction,
                 onUnmappableCharacter: CodingErrorAction,
                 replacementOpt: StringOption): Iterator[String] = {
    implicit val codec = Codec(encoding).
      onMalformedInput(onMalformedInput).
      onUnmappableCharacter(onUnmappableCharacter)
    if (replacementOpt.nonEmpty) {
      codec.decodingReplaceWith(replacementOpt.get)
    }
    Source.fromInputStream(
      new ByteArrayInputStream(
        repr.
          lineStream_!.
          iterator.
          mkString("\n").
          getBytes
      )
    ).getLines
  }
} 

//设置package包名称以及导入依赖的类
package io.plasmap.util.test

import io.plasmap.model.{OsmDenormalizedObject, OsmDenormalizedRelation}
import io.plasmap.parser.impl.OsmGeoJSONBoundaryParser

import scala.io.Codec
import scala.util.Random


object OsmTestData {

  private[this] def relations: PartialFunction[OsmDenormalizedObject, OsmDenormalizedRelation] = {
    case rel: OsmDenormalizedRelation => rel
  }

  private val testdataPath: String = "util/src/main/resources/"
  private[this] val cityEssenParser = OsmGeoJSONBoundaryParser(testdataPath + "city.essen.geojson")(Codec.UTF8)
  private[this] val cityMuehlheimParser = OsmGeoJSONBoundaryParser(testdataPath + "city.muehlheim.geojson")(Codec.UTF8)
  private[this] val cityDuisburgParser = OsmGeoJSONBoundaryParser(testdataPath + "city.duisburg.geojson")(Codec.UTF8)
  private[this] val districtsEssenParser = OsmGeoJSONBoundaryParser(testdataPath + "districts.essen.geojson")(Codec.UTF8)
  private[this] val districtsMuehlheimParser = OsmGeoJSONBoundaryParser(testdataPath + "districts.muehlheim.geojson")(Codec.UTF8)
  private[this] val districtsDuisburgParser = OsmGeoJSONBoundaryParser(testdataPath + "districts.duisburg.geojson")(Codec.UTF8)

  private[this] val cityEssen: List[OsmDenormalizedRelation] = (for {opt <- cityEssenParser
  } yield opt).flatten.collect(relations).toList
  private[this] val cityMuehlheim: List[OsmDenormalizedRelation] = (for {opt <- cityMuehlheimParser
  } yield opt).flatten.collect(relations).toList
  private[this] val cityDuisburg: List[OsmDenormalizedRelation] = (for {opt <- cityDuisburgParser
  } yield opt).flatten.collect(relations).toList

  val districtsEssen: List[OsmDenormalizedRelation] = (for {opt <- districtsEssenParser
  } yield opt).flatten.collect(relations).toList
  val districtsMuehlheim: List[OsmDenormalizedRelation] = (for {opt <- districtsMuehlheimParser
  } yield opt).flatten.collect(relations).toList
  val districtsDuisburg: List[OsmDenormalizedRelation] = (for {opt <- districtsDuisburgParser
  } yield opt).flatten.collect(relations).toList

  val essen = cityEssen.head
  val muehlheim = cityMuehlheim.head
  val duisburg = cityDuisburg.head

  val essenMuehlheimDistricts: List[OsmDenormalizedRelation] = Random.shuffle(districtsEssen.union(districtsMuehlheim))
  val duisburgMuehlheimDistricts: List[OsmDenormalizedRelation] = Random.shuffle(districtsDuisburg.union(districtsMuehlheim))
} 

//设置package包名称以及导入依赖的类
package com.scalafi.dynamics

import java.nio.file.{Files, Paths}

import com.scalafi.openbook.OpenBookMsg
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.joda.time.LocalDate
import org.joda.time.format.DateTimeFormat
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._
import scala.io.Codec

object OpenBook {
  private val log = LoggerFactory.getLogger(this.getClass)

  case class OpenBookFile(from: String, to: String, date: LocalDate, absolutePath: String)

  private[this] val openBookUltra = "openbookultra([A-Z]{2})_([A-Z]{1})([0-9]+)_1_of_1".r

  private[this] val dateFormat = DateTimeFormat.forPattern("YYYYMMdd")

  def openBookFiles(directory: String): Vector[OpenBookFile] = {
    val path = Paths.get(directory).toAbsolutePath
    assume(Files.isDirectory(path), s"Expected to see directory path: $directory")

    val files = Files.newDirectoryStream(path).
      iterator().asScala.map(_.toAbsolutePath.toString).
      map(absolutePath => (absolutePath, absolutePath.split("/").last)) collect {
      case (absolute, fileName @ openBookUltra(from, to, date)) =>
        OpenBookFile(from, to, dateFormat.parseLocalDate(date), absolute)
    }

    files.toVector
  }

  def orderLog(files: OpenBookFile*)(implicit sc: SparkContext): RDD[OpenBookMsg] = {
    val rdds = files map { file =>
      log.info(s"Load order log from Open Book file: ${file.absolutePath}")
      sc.parallelize(OpenBookMsg.iterate(file.absolutePath)(Codec.ISO8859).toSeq)
    }
    foldOrderLogs(rdds)
  }

  def orderLog(symbol: String, files: OpenBookFile*)(implicit sc: SparkContext): RDD[OpenBookMsg] = {
    val rdds = files map { file =>
      log.info(s"Load order log from Open Book file: ${file.absolutePath}")
      sc.parallelize(OpenBookMsg.iterate(file.absolutePath)(Codec.ISO8859).filter(_.symbol == symbol).toSeq)
    }
    foldOrderLogs(rdds)
  }

  private def foldOrderLogs(rdds: Iterable[RDD[OpenBookMsg]])(implicit sc: SparkContext): RDD[OpenBookMsg] =
    rdds.foldLeft(sc.emptyRDD[OpenBookMsg]: RDD[OpenBookMsg])(_ ++ _)

} 

//设置package包名称以及导入依赖的类
package com.scalafi.dynamics

import com.scalafi.dynamics.attribute.{LabeledPointsExtractor, MeanPriceMovementLabel}
import com.scalafi.openbook.OpenBookMsg
import org.scalatest.FlatSpec
import org.slf4j.LoggerFactory
import scala.concurrent.duration._

import scala.io.Codec

object Extractor {
  def extractor(symbol: String) = {
    import com.scalafi.dynamics.attribute.LabeledPointsExtractor._
    (LabeledPointsExtractor.newBuilder()
      += basic(_.askPrice(1))
      += basic(_.bidPrice(1))
      += basic(_.meanPrice)
      ).result(symbol, MeanPriceMovementLabel, LabeledPointsExtractor.Config(1.millisecond))
  }
}

class OrderLogFunctionsSpec extends FlatSpec with ConfiguredSparkContext {
  private val log = LoggerFactory.getLogger(classOf[OrderLogFunctionsSpec])

  lazy val orderLog = {
    implicit val codec = Codec.ISO8859
    val orders = OpenBookMsg.iterate(this.getClass.getResourceAsStream("/openbookultraAA_N20130403_1_of_1"))
    sc.parallelize(orders.toSeq)
  }

  val Symbol = "ANR"

  "OrderLogFunctions" should "get all symbols from order log" in {
    val symbols = orderLog.symbols()
    assert(symbols contains "ANR")
  }

  it should "count orders by symbol" in {
    val count = orderLog.countBySymbol()
    assert(count(Symbol) == 79)
    assert(count.toSeq.map(_._2).sum == orderLog.count())
  }
  
  it should "extract labeled data by symbol" in {
    val labeledOrderLog = orderLog.extractLabeledData(Symbol)(Extractor.extractor)
    assert(labeledOrderLog.size == 1)

    val ANROrderLog = labeledOrderLog.head
    assert(ANROrderLog.symbol == Symbol)

    val data = ANROrderLog.labeledPoints.collect().toVector
    log.info(s"Labeled data size = ${data.size}")
    // (Ask + Bid) / 2 == MeanPrice
    assert(data.size > 50)
    data foreach { labeledPoint =>
      val features = labeledPoint.features.toArray
      assert((features(0) + features(1)) / 2.0 == features(2))
    }
  }

} 

//设置package包名称以及导入依赖的类
package com.mchange.sc.v1.sbtethereum.repository

import java.io.{BufferedOutputStream,File,FileOutputStream,OutputStreamWriter,PrintWriter}
import java.util.Date
import java.text.SimpleDateFormat

import com.mchange.sc.v2.lang.borrow

import com.mchange.sc.v2.failable._

import com.mchange.sc.v1.consuela._
import com.mchange.sc.v1.consuela.ethereum.{EthHash,EthTransaction}

import scala.io.Codec

object TransactionLog {
  private val TimestampPattern = "yyyy-MM-dd'T'HH-mm-ssZ"

  lazy val File = Directory.map( dir => new java.io.File(dir, "transaction-log") )

  case class Entry( timestamp : Date, txn : EthTransaction.Signed, transactionHash : EthHash ) {
    override def toString() = {
      val ( ttype, payloadKey, payload ) = txn match {
        case m  : EthTransaction.Signed.Message          => ("Message", "data", m.data)
        case cc : EthTransaction.Signed.ContractCreation => ("ContractCreation", "init", cc.init)
      }
      val df = new SimpleDateFormat(TimestampPattern)
      val ts = df.format( timestamp )
      val first  = s"${ts}:type=${ttype},nonce=${txn.nonce.widen},gasPrice=${txn.gasPrice.widen},gasLimit=${txn.gasLimit.widen},value=${txn.value.widen},"
      val middle = if ( payload.length > 0 ) s"${payloadKey}=${payload.hex}," else ""
      val last   = s"v=${txn.v.widen},r=${txn.r.widen},s=${txn.s.widen},transactionHash=${transactionHash.bytes.hex}"
      first + middle + last
    }
  }

  def logTransaction( transaction : EthTransaction.Signed, transactionHash : EthHash ) : Unit = {
    File.flatMap { file =>
      Failable {
        val entry = TransactionLog.Entry( new Date(), transaction, transactionHash ) 
        borrow( new PrintWriter( new OutputStreamWriter( new BufferedOutputStream( new FileOutputStream( file, true ) ), Codec.UTF8.charSet ) ) )( _.println( entry ) )
      }
    }.get // Unit or vomit Exception
  }
}