本文整理汇总了Scala中grizzled.slf4j.Logging类的典型用法代码示例。如果您正苦于以下问题:Scala Logging类的具体用法?Scala Logging怎么用?Scala Logging使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Logging类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Scala代码示例。
示例1: WriteLines
//设置package包名称以及导入依赖的类
package org.hammerlab.coverage.utils
import java.io.PrintWriter
import grizzled.slf4j.Logging
import org.hammerlab.paths.Path
object WriteLines
extends Logging {
def apply(path: Path, strs: Iterator[String], force: Boolean): Unit =
if (!force && path.exists) {
logger.info(s"Skipping $path, already exists")
} else {
val os = new PrintWriter(path.outputStream)
strs.foreach(os.println)
os.close()
}
}
开发者ID:hammerlab,项目名称:coverage-depth,代码行数:19,代码来源:WriteLines.scala
示例2: WriteRDD
//设置package包名称以及导入依赖的类
package org.hammerlab.coverage.utils
import grizzled.slf4j.Logging
import org.apache.spark.rdd.RDD
import org.hammerlab.csv._
import org.hammerlab.paths.Path
import scala.reflect.runtime.universe.TypeTag
object WriteRDD
extends Logging {
def apply[T <: Product : TypeTag](path: Path, rdd: RDD[T], force: Boolean): Unit = {
val csvLines = rdd.mapPartitions(_.toCSV(includeHeaderLine = false))
(path.exists, force) match {
case (true, true) ?
logger.info(s"Removing $path")
path.delete(recursive = true)
csvLines.saveAsTextFile(path.toString)
case (true, false) ?
logger.info(s"Skipping $path, already exists")
case _ ?
csvLines.saveAsTextFile(path.toString)
}
}
}
开发者ID:hammerlab,项目名称:coverage-depth,代码行数:26,代码来源:WriteRDD.scala
示例3: ReqRealTimeBarsHandler
//设置package包名称以及导入依赖的类
package name.kaeding.fibs
package ib
package impl
package handlers
import java.util.concurrent.CountDownLatch
import scalaz._, Scalaz._
import scalaz.concurrent._
import messages._
import contract._
import java.util.concurrent.{ LinkedBlockingQueue, BlockingQueue }
import com.ib.client.EClientSocket
import com.github.nscala_time.time.Imports._
import grizzled.slf4j.Logging
class ReqRealTimeBarsHandler(security: Stock ,
ibActor: Actor[FibsPromiseMessage \/ IBMessage],
tickerId: Int, socket: EClientSocketLike) extends FibsPromise[CloseableStream[RealTimeBar]] with Logging {
private[this] val TickerId = tickerId
val latch = new CountDownLatch(0) // don't need to block
val actor = Actor[IBMessage] {
case RealTimeBarResp(TickerId, time, open, high, low, close, volume, count, wap) ?
queue.add(RealTimeBar(new DateTime(time * 1000), open, high, low, close, volume, count, wap).some)
case _ ? ???
}
val barHandler: PartialFunction[IBMessage, Unit] = {
case [email protected](TickerId, time, open, high, low, close, volume, count, wap) ? actor ! m
}
val patterns = List(barHandler)
private[this] def toStream: EphemeralStream[RealTimeBar] = {
val ret: EphemeralStream[RealTimeBar] = queue.take match {
case Some(d) ? EphemeralStream.cons(d, toStream)
case None ? EphemeralStream.emptyEphemeralStream
}
ret
}
private[this] val queue: BlockingQueue[Option[RealTimeBar]] =
new LinkedBlockingQueue[Option[RealTimeBar]]()
private[this] def closeStream = {
queue add None
socket.cancelRealTimeBars(tickerId)
ibActor ! UnregisterFibsPromise(this).left
}
def get = new CloseableStream[RealTimeBar] {
def close = closeStream
lazy val as = toStream
}
}
开发者ID:carrot-garden,项目名称:vendor_ibrk_fibs-scala,代码行数:49,代码来源:ReqRealTimeBarsHandler.scala
示例4: ReqMarketTickDataStreamHandler
//设置package包名称以及导入依赖的类
package name.kaeding.fibs
package ib
package impl
package handlers
import java.util.concurrent.CountDownLatch
import scalaz._, Scalaz._
import scalaz.concurrent._
import messages._
import contract._
import java.util.concurrent.{ LinkedBlockingQueue, BlockingQueue }
import com.ib.client.EClientSocket
import com.github.nscala_time.time.Imports._
import grizzled.slf4j.Logging
class ReqMarketTickDataStreamHandler(security: Stock ,
ibActor: Actor[FibsPromiseMessage \/ IBMessage],
tickerId: Int, socket: EClientSocketLike) extends FibsPromise[CloseableStream[MarketTickDataResult]] with Logging {
private[this] val TickerId = tickerId
val latch = new CountDownLatch(0) // don't need to block
private[this] val RTVolumePattern = "(\\d+\\.?\\d*);(\\d+);(\\d+);(\\d+);(\\d+\\.?\\d*);(true|false)".r
val actor = Actor[IBMessage] {
case TickString(TickerId, RTVolume, v) ?
parseInput(v).cata(some = t => queue.add(t.some),
none = warn(s"error parsing tick data: $v"))
case _ ? ???
}
def parseInput(s: String) = s match {
case RTVolumePattern(p, s, t, v, w, f) =>
(p.parseDouble.toOption |@|
s.parseInt.toOption |@|
v.parseInt.toOption |@|
t.parseLong.toOption |@|
w.parseDouble.toOption |@|
f.parseBoolean.toOption)(MarketTickDataResult.apply)
case _ => none
}
val stringHandler: PartialFunction[IBMessage, Unit] = {
case [email protected](tickerId, RTVolume, _) ? actor ! m
}
val patterns = List(stringHandler)
private[this] val queue: BlockingQueue[Option[MarketTickDataResult]] =
new LinkedBlockingQueue[Option[MarketTickDataResult]]()
private[this] def closeStream = {
queue add None
socket.cancelMktData(TickerId)
ibActor ! UnregisterFibsPromise(this).left
}
private[this] def toStream: EphemeralStream[MarketTickDataResult] = {
val ret: EphemeralStream[MarketTickDataResult] = queue.take match {
case Some(d) ? EphemeralStream.cons(d, toStream)
case None ? EphemeralStream.emptyEphemeralStream
}
ret
}
def get = new CloseableStream[MarketTickDataResult] {
def close = closeStream
lazy val as = toStream
}
}
开发者ID:carrot-garden,项目名称:vendor_ibrk_fibs-scala,代码行数:61,代码来源:ReqMarketTickDataStreamHandler.scala
示例5: Actors
//设置package包名称以及导入依赖的类
package com.amadornes.modcast.bot
import akka.actor.{ActorSystem, Props}
import akka.stream.ActorMaterializer
import com.amadornes.modcast.bot.discord.commands._
import com.amadornes.modcast.bot.discord.{CommandDispatcher, DiscordActor, DiscordNotifier}
import com.amadornes.modcast.bot.helpers.ShutdownHelper
import com.amadornes.modcast.bot.servers.MCWhitelistServer
import grizzled.slf4j.Logging
object Actors extends Logging {
implicit val system = ActorSystem()
implicit val materializer = ActorMaterializer()
ShutdownHelper.registerHandler(() => {
info("Shutting down actor system...")
system.terminate()
})
object servers {
val MCWhitelistServer = system.actorOf(Props[MCWhitelistServer])
}
object discord {
val actor = system.actorOf(Props[DiscordActor])
val notifier = system.actorOf(Props[DiscordNotifier])
object commands {
val commandDispatcher = system.actorOf(Props[CommandDispatcher])
val commandHandlers = Array(
system.actorOf(Props[PingCommand]),
system.actorOf(Props[WhoamiCommand]),
system.actorOf(Props[GuestCommand]),
system.actorOf(Props[StreamCommand]),
system.actorOf(Props[ShutdownCommand]),
system.actorOf(Props[WhitelistCommand]),
system.actorOf(Props[UserCommand])
)
}
}
}
开发者ID:Modcast,项目名称:ModcastBot,代码行数:45,代码来源:Actors.scala
示例6: Main
//设置package包名称以及导入依赖的类
package com.amadornes.modcast.bot
import com.amadornes.modcast.bot.database.{DB, Permission, UserPermission}
import com.amadornes.modcast.bot.discord.DiscordMessages
import com.amadornes.modcast.bot.servers.http.RTMPControlServer
import grizzled.slf4j.Logging
import scala.collection.JavaConversions._
object Main extends App with Logging {
info("Bot is now loading!")
info("Starting discord...")
Actors.discord.actor ! DiscordMessages.Startup()
info("Forcing admins...")
Configuration.config.getStringList("users.forcedAdmins").foreach(admin =>
DB.save(DB.query[UserPermission].whereEqual("user", admin).fetchOne().getOrElse(UserPermission(user = admin, permission = Permission.NONE.getLevel)).copy(permission = Permission.ADMIN.getLevel))
)
info("Starting RTMP control...")
RTMPControlServer.start()
info("Booping MCWhitelistServer...")
Actors.servers.MCWhitelistServer ! "boop"
info("We are online!")
}
开发者ID:Modcast,项目名称:ModcastBot,代码行数:28,代码来源:Main.scala
示例7: MCWhitelistServer
//设置package包名称以及导入依赖的类
package com.amadornes.modcast.bot.servers
import java.net.InetSocketAddress
import akka.actor.{Actor, ActorRef}
import akka.io.{IO, Tcp}
import akka.util.ByteString
import com.amadornes.modcast.bot.Configuration
import grizzled.slf4j.Logging
import scala.collection.mutable.ArrayBuffer
class MCWhitelistServer extends Actor with Logging {
import Tcp._
import context.system
IO(Tcp) ! Bind(self, new InetSocketAddress(Configuration.config.getString("mc.host"), Configuration.config.getInt("mc.port")))
val connections = new ArrayBuffer[ActorRef]()
def receive = {
case str: String =>
//Ignored
case [email protected](localAddress) =>
info("MC Whitelist TCP Server is online and listening.")
case CommandFailed(_: Bind) =>
error("Bind failed. MC Whitelist Server is not functional.")
context stop self
case [email protected](remote, local) =>
if (Configuration.config.getStringList("mc.acceptedIPs").contains(remote.getAddress.getHostAddress)) {
sender() ! Register(self)
connections += sender()
} else {
warn(s"Refusing connection from ${remote.getAddress.getHostAddress}")
sender() ! Close
}
case MCWhitelistServer.WhitelistUser(userID) =>
for (connection <- connections)
connection ! Write(ByteString(s"W$userID\n"))
case MCWhitelistServer.UnWhitelistUser(userID) =>
for (connection <- connections)
connection ! Write(ByteString(s"U$userID\n"))
}
}
object MCWhitelistServer {
case class WhitelistUser(id: String)
case class UnWhitelistUser(id: String)
}
开发者ID:Modcast,项目名称:ModcastBot,代码行数:59,代码来源:MCWhitelistServer.scala
示例8: TestItems
//设置package包名称以及导入依赖的类
package org.leialearns.crystallize.item
import grizzled.slf4j.Logging
import org.leialearns.crystallize.util.LoggingConfiguration
import org.scalatest.FunSuite
class TestItems extends FunSuite with LoggingConfiguration with Logging {
test("Items") {
info("\n\nTest items")
val actions = Category.getCategory("actions")
assert(actions eq Category.getCategory("actions"))
val responses = Category.getCategory("responses")
assert(actions != responses)
val dark = Item.getItem(responses, "dark")
assert(dark eq Item.getItem(responses, "dark"))
val light = Item.getItem(responses, "light")
assert(dark != light)
val left = Item.getItem(actions, "left")
assert(left eq Item.getItem(actions, "left"))
val right = Item.getItem(responses, "right")
assert(left != right)
val justLeft = Node.getNode(left)
assert(justLeft eq Node.getNode(left))
val leftDark = Node.getNode(justLeft, dark)
assert(leftDark eq Node.getNode(justLeft, dark))
}
}
开发者ID:jeroenvanmaanen,项目名称:crystallize,代码行数:29,代码来源:TestItems.scala
示例9: EventsSource
//设置package包名称以及导入依赖的类
package me.rexim.issuestant.github
import io.circe._
import io.circe.generic.auto._
import io.circe.parser._
import io.circe.syntax._
import scalaz.stream._
import scalaz.concurrent._
import org.http4s._
import org.http4s.util._
import org.http4s.client._
import org.http4s.circe._
import org.http4s.Http4s._
import org.http4s.Status._
import org.http4s.Method._
import org.http4s.EntityDecoder
import scalaz._
import me.rexim.issuestant.github.model._
import grizzled.slf4j.Logging
// $COVERAGE-OFF$
class EventsSource[E] (etagPolling: EtagPolling) extends Logging {
def events(implicit decoder: Decoder[E]): Process[Task, E] =
etagPolling.responses
.flatMap(extractEvents)
.map((e) => { info(s"New GitHub event: ${e}"); e })
private def extractEvents(response: Response)
(implicit decoder: Decoder[E]): Process[Task, E] = {
implicit val eventHttp4s = jsonOf[E]
implicit val listEventHttp4s = jsonOf[List[E]]
Process.emitAll(response.as[List[E]].run)
}
}
// $COVERAGE-ON$
开发者ID:tsoding,项目名称:Issuestant,代码行数:43,代码来源:EventsSource.scala
示例10: ApiErrorParser
//设置package包名称以及导入依赖的类
package io.corbel.sdk.error
import com.ning.http.client.Response
import grizzled.slf4j.Logging
import org.json4s.DefaultFormats
import org.json4s.JsonAST.JValue
import org.json4s.native.JsonMethods._
import scala.util.{Success, Try}
class ApiErrorParser[T](f: Response => T) extends (Response => Either[ApiError, T]) with Logging {
override def apply(res: Response): Either[ApiError, T] = res.getStatusCode match {
case ok: Int if ok / 100 == 2 => Right(f(res))
case other: Int => Left(apiErrorBody(res))
}
def apiErrorBody(res: Response): ApiError = {
val apiError = ApiError(status = res.getStatusCode)
if(!res.hasResponseBody){
apiError
}
else {
Try(parse(res.getResponseBodyAsStream)) match {
case Success(json) => completeFromJson(apiError, json)
case _ =>
warn(s"Corbel error message without expected JSON body: ${res.getStatusCode} : ${res.getResponseBody}")
apiError.copy(message = Option(res.getResponseBody))
}
}
}
def completeFromJson(apiError: ApiError, json: JValue): ApiError = {
implicit val format = DefaultFormats
val c = (json \ "error").extractOpt[String]
val m = (json \ "errorDescription").extractOpt[String]
apiError.copy(errorCode = c, message = m)
}
}
开发者ID:devialab,项目名称:corbel-sdk,代码行数:41,代码来源:ApiErrorParser.scala
示例11: MapBoard
//设置package包名称以及导入依赖的类
package io.makana.hexwar.engine.domain.model
import grizzled.slf4j.Logging
import io.makana.hexwar.engine.domain.vector.Vectr
import scala.collection.mutable
class MapBoard(val size: Vectr) extends Logging {
val hexes = new Array[Hex](size.product)
val entities = new mutable.HashMap[String, Entity]()
def setHexAt(value: Hex, x: Int, y: Int): Unit = {
if (x >= size.x || x < 0) {
throw new IllegalArgumentException("invalid x coordinates")
}
if (y >= size.x || y < 0) {
throw new IllegalArgumentException("invalid y coordinates")
}
hexes(getPos(x,y)) = value
}
def getHexAt(x: Int, y: Int) : Hex = {
if (x >= size.x || x < 0) {
throw new IllegalArgumentException("invalid x coordinates")
}
if (y >= size.x || y < 0) {
throw new IllegalArgumentException("invalid y coordinates")
}
hexes(getPos(x,y))
}
private def getPos(x: Int, y: Int) : Int = {
size.y * x + y
}
}
sealed trait Hex
case class Plains() extends Hex
case class Road() extends Hex
case class Swamp() extends Hex
case class Woods() extends Hex
case class Stream() extends Hex
case class River() extends Hex
case class Beach() extends Hex
case class Ocean() extends Hex
case class Town() extends Hex
case class Hill() extends Hex
case class Hilltop() extends Hex
case class TemporaryBridge() extends Hex
case class PermanentBridge() extends Hex
开发者ID:brmakana,项目名称:hexwar,代码行数:52,代码来源:MapBoard.scala
示例12: Authorized
//设置package包名称以及导入依赖的类
package controllers.hocs
import scala.concurrent.Future
import play.api.mvc.{Action, Request, Result}
import play.api.mvc.Results.Unauthorized
import grizzled.slf4j.Logging
case class Authorized[A](action: Action[A]) extends Action[A] with Logging {
lazy val parser = action.parser
def apply(request: Request[A]): Future[Result] = {
request.getQueryString("alf_ticket") match {
case Some("TICKET") =>
action(request)
case _ =>
info(s"Not authorized")
Future successful Unauthorized
}
}
}
开发者ID:UKHomeOffice,项目名称:hocs-fake,代码行数:22,代码来源:Authorized.scala
示例13: HocsController
//设置package包名称以及导入依赖的类
package controllers.hocs
import javax.inject.Singleton
import play.api.libs.json.Json
import play.api.mvc.{Action, Controller}
import grizzled.slf4j.Logging
import models.Credentials
@Singleton
class HocsController extends Controller with Logging {
def authorize = Action(parse.json) { request =>
val credentials = request.body.as[Credentials]
info(s"Authorizing ${credentials.userName}")
(credentials.userName, credentials.password) match {
case ("userName", "password") =>
info(s"${credentials.userName} has been authorized")
Ok(Json.obj("data" -> Json.obj("ticket" -> "TICKET")))
case _ =>
error(s"${credentials.userName} failed authorization")
Unauthorized
}
}
def createCase = Authorized {
Action(parse.multipartFormData) { request =>
val files = request.body.files
info(s"Number of documents received = ${files.size}")
Ok(Json.obj(
"documents-received" -> Json.obj("count" -> files.size, "types" -> files.map(_.contentType).mkString(", ")),
"caseRef" -> "CASE_REF"
))
}
}
def document = Authorized {
Action(parse.multipartFormData) { request =>
val files = request.body.files
info(s"Number of documents received = ${files.size}")
Ok(Json.obj(
"documents-received" -> Json.obj("count" -> files.size, "types" -> files.map(_.contentType).mkString(", "))
))
}
}
}
开发者ID:UKHomeOffice,项目名称:hocs-fake,代码行数:50,代码来源:HocsController.scala
示例14: EventService
//设置package包名称以及导入依赖的类
package trove.core.event
import akka.actor.{Actor, ActorRef, ActorSystem, Props}
import grizzled.slf4j.Logging
import scala.concurrent.Await
import scala.concurrent.duration.Duration
object EventService extends Logging {
// ActorSystem is a heavy object: create only one per application
// http://doc.example.io/docs/example/snapshot/scala/actors.html
logger.debug("Starting actor system")
private[this] val system = ActorSystem("actorsystem")
@volatile private[this] var subscriptions = Map.empty[EventListener,ActorRef]
def publish(event: Event) {
system.eventStream.publish(UntypedEvent(event))
}
def shutdown(): Unit = {
logger.debug("Shutting down actor service")
Await.result(system.terminate(), Duration.Inf)
}
private[event] def subscribeEvents(listener: EventListener) {
if(!subscriptions.contains(listener)) {
logger.debug(s"Adding subscription for listener ${listener.toString} : ${listener.getClass.getName}")
val props = Props(classOf[Subscriber], listener)
val subscriber = system.actorOf(props)
system.eventStream.subscribe(subscriber, classOf[UntypedEvent])
subscriptions += listener -> subscriber
logger.debug(s"Subscriber map size: ${subscriptions.size}")
}
else {
logger.warn(s"Listener ${listener.toString} : ${listener.getClass.getName} is already subscribed for events")
}
}
private[event] def unsubscribeEvents(listener: EventListener) {
subscriptions.get(listener).foreach { _ =>
logger.debug(s"Removing subscription for listener ${listener.toString} : ${listener.getClass.getName}")
system.stop(_)
}
subscriptions -= listener
logger.debug(s"Subscriber map size: ${subscriptions.size}")
}
sealed case class UntypedEvent(event: Event)
sealed class Subscriber(listener: EventListener) extends Actor {
override def receive: PartialFunction[Any, Unit] = {
case UntypedEvent(event) => listener.onEvent(event)
}
}
}
开发者ID:emanchgo,项目名称:trove,代码行数:58,代码来源:EventService.scala
示例15: AkkaSupervision
//设置package包名称以及导入依赖的类
package trove.core.event
import akka.actor.SupervisorStrategy.{Resume, Stop}
import akka.actor.{ActorInitializationException, ActorKilledException, OneForOneStrategy, SupervisorStrategy, SupervisorStrategyConfigurator}
import grizzled.slf4j.Logging
class AkkaSupervision extends SupervisorStrategyConfigurator with Logging {
override def create(): SupervisorStrategy = {
logger.debug("Creating custom akka supervisor strategy")
OneForOneStrategy() {
case _: ActorInitializationException => Stop
case _: ActorKilledException => Stop
case e: Exception =>
logger.error("Exception in event listener", e)
Resume
}
}
}
开发者ID:emanchgo,项目名称:trove,代码行数:19,代码来源:AkkaSupervision.scala
示例16: CryptoFilter
//设置package包名称以及导入依赖的类
package uk.gov.homeoffice.aws.sqs
import java.security.MessageDigest
import scala.util.{Failure, Success}
import org.json4s.jackson._
import grizzled.slf4j.Logging
import uk.gov.homeoffice.crypt.{Crypto, Secrets}
class CryptoFilter(implicit secrets: Secrets) extends (Message => Option[Message]) with Crypto with Logging {
def apply(msg: Message): Option[Message] = decrypt(parseJson(msg.content)) match {
case Success(a) =>
val sqsMessage = new com.amazonaws.services.sqs.model.Message()
sqsMessage.setAttributes(msg.sqsMessage.getAttributes)
sqsMessage.setBody(a)
sqsMessage.setMD5OfBody(new String(MessageDigest.getInstance("MD5").digest(a.getBytes)))
sqsMessage.setMessageId(msg.messageID)
sqsMessage.setMD5OfMessageAttributes(msg.sqsMessage.getMD5OfMessageAttributes)
sqsMessage.setReceiptHandle(msg.sqsMessage.getReceiptHandle)
Some(Message(sqsMessage))
case Failure(t) =>
warn(s"Failed to decrypt message because of: ${t.getMessage}, where given message was: $msg")
None
}
}
开发者ID:UKHomeOffice,项目名称:aws-scala-lib,代码行数:26,代码来源:CryptoFilter.scala
示例17: create
//设置package包名称以及导入依赖的类
package uk.gov.homeoffice.aws.sqs
import grizzled.slf4j.Logging
trait QueueCreation extends Logging {
def create(queue: Queue)(implicit sqsClient: SQSClient): Queue = {
def createQueue(queueName: String) = try {
info(s"Creating queue $queueName")
sqsClient createQueue queueName
} catch {
case t: Throwable => warn(s"Application has not created queue $queueName - Exception: ${t.getMessage}")
}
createQueue(queue.queueName)
createQueue(queue.errorQueueName)
queue
}
}
开发者ID:UKHomeOffice,项目名称:aws-scala-lib,代码行数:19,代码来源:QueueCreation.scala
示例18: SQSClient
//设置package包名称以及导入依赖的类
package uk.gov.homeoffice.aws.sqs
import java.net.URL
import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.AWSCredentials
import com.amazonaws.services.sqs.AmazonSQSClient
import grizzled.slf4j.Logging
class SQSClient(val sqsHost: URL, credentials: AWSCredentials)(implicit clientConfiguration: ClientConfiguration = new ClientConfiguration) extends AmazonSQSClient(credentials, clientConfiguration) with Logging {
val Host = """^(http[s]?):\/?\/?([^:\/\s]+):?(\d*).*""".r
sqsHost.toString match {
case Host(protocol, "localhost", port) =>
info(s"Configuring endpoint as $protocol://localhost:$port")
setEndpoint(s"$protocol://localhost:$port")
case Host(protocol, host, port) if port == "" =>
info(s"Configuring endpoint as $host")
setEndpoint(s"$host")
case Host(protocol, host, port) =>
info(s"Configuring endpoint as $protocol://$host:$port")
setEndpoint(s"$protocol://$host:$port")
}
def clientConfig = clientConfiguration
}
开发者ID:UKHomeOffice,项目名称:aws-scala-lib,代码行数:28,代码来源:SQSClient.scala
示例19: ConvertPstToParquet
//设置package包名称以及导入依赖的类
package com.hugolinton
import com.hugolinton.model.EmailModel
import com.pff.{PSTFile, PSTMessage}
import grizzled.slf4j.Logging
import org.apache.spark.SparkContext
import org.apache.spark.sql.{Row, SQLContext}
import scala.collection.JavaConversions._
object ConvertPstToParquet extends Logging {
def main(args: Array[String]): Unit = {
if(args.length != 3){
Console.err.println("Usage: ConvertPstToParquet <PST File Path> <Export Folder><Hadoop Home Directory>")
}
val Array(pstFile, parquetFolder, hadoopHome) = args
System.setProperty("hadoop.home.dir", hadoopHome)
val sparkConf = new org.apache.spark.SparkConf().setAppName("com.hugolinton.pst-to-parquet").setMaster("local");
val sparkContext = new SparkContext(sparkConf)
val sqlContext = new org.apache.spark.sql.SQLContext(sparkContext)
pstToParquet(sparkContext,sqlContext,pstFile,parquetFolder)
}
def pstToParquet(sparkContext : SparkContext,sqlContext : SQLContext, filePath : String, outputFolder : String) : String = {
val pstFile = new PSTFile(filePath)
val expandedPSTFiles = new ExpandPSTFiles()
expandedPSTFiles.processFolder(pstFile.getRootFolder)
val emails = expandedPSTFiles.getEmails.toSet.map( (e: PSTMessage) => {
EmailModel.pstToModel(e)
}).toSeq
if(emails.size != expandedPSTFiles.getEmails.size()){
error("Some records were lost!")
}
val rdd = sparkContext.parallelize(emails).map(email => Row(email.id, email.emailBody, email.sentTo, email.ccTo))
val df = sqlContext.createDataFrame(rdd, EmailModel.schema)
val exportLocation = outputFolder + "\\" + System.currentTimeMillis.toString
info("Export Location: " + exportLocation)
df.write.parquet(exportLocation)
exportLocation
}
}
开发者ID:HugoLinton,项目名称:PST-to-Parquet,代码行数:53,代码来源:ConvertPstToParquet.scala
示例20: Companion
//设置package包名称以及导入依赖的类
package com.tierline.scala.activemodel.util
import grizzled.slf4j.Logging
object Companion extends Logging {
def of[T: Manifest]: Option[AnyRef] = try {
val classOfT = implicitly[Manifest[T]].runtimeClass
val companionClassName = classOfT.getName + "$"
val companionClass = Class.forName(companionClassName)
val moduleField = companionClass.getField("MODULE$")
Some(moduleField.get(null))
} catch {
case e: Throwable => {
debug(s"$e ${e.getCause}")
None
}
}
}
开发者ID:tierline,项目名称:scala-activemodel,代码行数:19,代码来源:Companion.scala
注:本文中的grizzled.slf4j.Logging类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论