//设置package包名称以及导入依赖的类
package uk.co.telegraph.sbt.resolver

import java.net.URL

import sbt.CrossVersion._
import sbt._

import scala.xml.XML

trait ModuleResolver {

  implicit val repository  :Repository
  implicit val scalaVersion:String
  val module:ModuleID

  lazy val groupId      = module.organization
  lazy val artifactId   = module.crossVersion match {
    case Disabled => module.name
    case _        => s"${module.name}_$scalaVersion"
  }
  lazy val revision   = module.revision

  private [resolver] lazy val metadata = loadMetadata


  //Check if version exists
  def checkVersion(version:String) =
    metadata.versions.contains(version)

  def resolveVersion(versionOpt:Option[String] = None):ModuleResolver = {
    val resolvedVersion:Option[String] = versionOpt.orElse( Some(revision) )
    val versionLabel = resolvedVersion
      .map({
        case LatestVersionLabel => metadata.latest
        case ReleaseVersionLabel => metadata.release
        case version   => version
      })
      .filter(checkVersion)
      .getOrElse(metadata.latest)

    fromModuleIdAndMetadata(module.copy(revision = versionLabel), metadata)
  }

  //Get remove Jar
  def remoteJar:URL = artifactUrl

  private [resolver] def metadataUrl:URL =
    url(s"${repository.url}/${groupId.replaceAll("\\.", "/")}/$artifactId/${repository.metadata}")

  private [resolver] def artifactUrl:URL =
    url(s"${repository.url}/${groupId.replaceAll("\\.", "/")}/$artifactId/$revision/$artifactId-$revision.jar")

  private [resolver] def loadMetadata:ModuleMetadata = {
    val versioning = XML.load( metadataUrl ) \\ "versioning"
    val versions   = (versioning \ "versions" \ "version").map(_.text)
    val release    = (versioning \ "release").text
    val latest     = (versioning \ "latest").text

    ModuleMetadata(versions, release, latest)
  }
} 

//设置package包名称以及导入依赖的类
package com.galacticfog.gestalt.lambda.impl

import java.io.File

import java.io.IOException
import java.lang.reflect.Method
import java.net.URL
import java.net.URLClassLoader
import java.util.Iterator
import java.util.ServiceLoader

case class ClassPathExistsException( msg : String ) extends Exception

object DynLoader {

  def extendClasspath( dir : File, inLoader : ClassLoader ) = {
    try {

      val sysLoader : URLClassLoader = inLoader match {
        case u : URLClassLoader => u
        case _ => throw new Exception( "Cast Exception" )
      }

      val urls : Seq[URL] = sysLoader.getURLs()
      val udir = dir.toURI().toURL()

      val udirs = udir.toString()
      urls.find( u => u.toString().equalsIgnoreCase(udirs) ) match {
        case Some(s) => throw new ClassPathExistsException( "class path exists" )
        case None => {}
      }

      val sysClass = classOf[URLClassLoader]

      val method : Method = sysClass.getDeclaredMethod("addURL", classOf[URL] )
      method.setAccessible(true)
      val udirObj = udir match {
        case o : Object => o
        case _ => throw new Exception( "impossible" )
      }

      method.invoke(sysLoader, udirObj )
      println( "Loaded " + udirs + " dynamically...")
    }
    catch {
      case cpe : ClassPathExistsException => {
        println( "class path exists, ignoring" )
      }
      case t : Throwable  => {
        t.printStackTrace();
      }
    }
  }
} 

//设置package包名称以及导入依赖的类
package mesosphere.marathon.upgrade

import java.net.URL

import akka.actor.Status.Failure
import akka.actor.{ Actor, Props }
import akka.pattern.pipe
import mesosphere.marathon.ResolveArtifactsCanceledException
import mesosphere.marathon.io.storage.StorageProvider
import mesosphere.marathon.io.{ CancelableDownload, PathFun }
import mesosphere.util.Logging

import scala.concurrent.Promise

private[this] class ResolveArtifactsActor(
  url2Path: Map[URL, String],
  promise: Promise[Boolean],
  storage: StorageProvider)
    extends Actor
    with PathFun
    with Logging {

  import mesosphere.marathon.upgrade.ResolveArtifactsActor.DownloadFinished

  // all downloads that have to be performed by this actor
  var downloads = url2Path.map { case (url, path) => new CancelableDownload(url, storage, path) }

  override def preStart(): Unit = {
    import context.dispatcher
    downloads.map(_.get.map(DownloadFinished) pipeTo self)
    if (url2Path.isEmpty) promise.success(true) // handle empty list
  }

  override def postStop(): Unit = {
    downloads.foreach(_.cancel()) // clean up not finished artifacts
  }

  override def receive: Receive = {
    case DownloadFinished(download) =>
      downloads = downloads.filter(_ != download)
      if (downloads.isEmpty) promise.success(true)
    case Failure(ex) =>
      log.warn("Can not resolve artifact", ex) // do not fail the promise!
    case DeploymentActor.Shutdown =>
      if (!promise.isCompleted)
        promise.tryFailure(new ResolveArtifactsCanceledException("Artifact Resolving has been cancelled"))
      context.stop(self)
  }
}

object ResolveArtifactsActor {
  def props(
    url2Path: Map[URL, String],
    promise: Promise[Boolean],
    storage: StorageProvider): Props = Props(new ResolveArtifactsActor(url2Path, promise, storage))

  case class DownloadFinished(download: CancelableDownload)
} 

//设置package包名称以及导入依赖的类
package com.aluxian.tweeather.scripts

import java.net.URL
import java.util.zip.ZipInputStream

import org.apache.hadoop.fs.Path
import org.apache.spark.Logging


object Sentiment140Downloader extends Script with Logging {

  val downloadUrl = "http://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip"

  override def main(args: Array[String]) {
    super.main(args)

    logInfo(s"Downloading sentiment140 dataset from $downloadUrl")
    val zip = new ZipInputStream(new URL(downloadUrl).openStream())
    val buffer = new Array[Byte](4 * 1024)

    Stream.continually(zip.getNextEntry)
      .takeWhile(_ != null)
      .foreach { entry =>
        val fileName = entry.getName
        val out = hdfs.create(new Path(s"tw/sentiment/140/downloaded/$fileName"))
        logInfo(s"Downloading $fileName")

        Stream.continually(zip.read(buffer))
          .takeWhile(_ != -1)
          .foreach { count =>
            out.write(buffer, 0, count)
          }

        out.close()
      }

    zip.close()
    logInfo("Downloading finished")
    sc.stop()
  }

} 

//设置package包名称以及导入依赖的类
package pubsub

import java.net.ServerSocket
import java.net.Socket
import java.io.BufferedReader
import java.io.InputStreamReader
import java.net.URL

import java.util.concurrent.Executors
import scala.concurrent.JavaConversions._
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

import pubsub.collection._
import pubsub.command._
import pubsub.network.TCPReader

object Server extends App {
  val port = 7676
  val maxWorkers = 12
  val bufferSize = 20
  val socket = new ServerSocket(port)
  try {
    val whatismyip = new URL("http://checkip.amazonaws.com")
    val in = new BufferedReader(new InputStreamReader(whatismyip.openStream()));
    val serverIP = in.readLine()
    println(s"Connect to $serverIP (or `localhost`), port $port with `telnet` to join this server")
  } catch  {
    case e: Exception =>
      println("There is a problem with your internet connection, you can only access it via localhost")
  }

  val buffer = new BoundedBuffer[Command](20)
  val commandHandlers = for{
    i <- 0 until maxWorkers
  } yield {
    Future {
      new CommandHandler(buffer).handle()
    }
  }
  val threadPool = Executors.newFixedThreadPool(maxWorkers)

  var clientId = 0
  while(true) {
    val client = socket.accept();
    val cid = clientId
    clientId += 1
    Future{
      new TCPReader(clientId, client, buffer).read()
    }(threadPool)
  }
} 

//设置package包名称以及导入依赖的类
package uk.gov.hmrc.agentmapping.support

import java.net.URL

import com.github.tomakehurst.wiremock.WireMockServer
import com.github.tomakehurst.wiremock.client.WireMock.{configureFor, reset}
import com.github.tomakehurst.wiremock.core.WireMockConfiguration
import com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite}
import uk.gov.hmrc.play.it.Port.randomAvailable

case class WireMockBaseUrl(value: URL)

object WireMockSupport {
  // We have to make the wireMockPort constant per-JVM instead of constant
  // per-WireMockSupport-instance because config values containing it are
  // cached in the GGConfig object
  private lazy val wireMockPort = randomAvailable
}

trait WireMockSupport extends BeforeAndAfterAll with BeforeAndAfterEach{
  me: Suite =>

  val wireMockPort: Int = WireMockSupport.wireMockPort
  val wireMockHost = "localhost"
  val wireMockBaseUrlAsString = s"http://$wireMockHost:$wireMockPort"
  val wireMockBaseUrl = new URL(wireMockBaseUrlAsString)
  protected implicit val implicitWireMockBaseUrl = WireMockBaseUrl(wireMockBaseUrl)

  protected def basicWireMockConfig(): WireMockConfiguration = wireMockConfig()

  private val wireMockServer = new WireMockServer(basicWireMockConfig().port(wireMockPort))

  override protected def beforeAll(): Unit = {
    super.beforeAll()
    configureFor(wireMockHost, wireMockPort)
    wireMockServer.start()
  }

  override protected def afterAll(): Unit = {
    wireMockServer.stop()
    super.afterAll()
  }

  override protected def beforeEach(): Unit = {
    super.beforeEach()
    reset()
  }
} 

//设置package包名称以及导入依赖的类
package uk.gov.hmrc.agentmappingfrontend.support

import java.net.URL

import com.github.tomakehurst.wiremock.WireMockServer
import com.github.tomakehurst.wiremock.client.WireMock.{configureFor, reset}
import com.github.tomakehurst.wiremock.core.WireMockConfiguration
import com.github.tomakehurst.wiremock.core.WireMockConfiguration._
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite}
import uk.gov.hmrc.play.it.Port

case class WireMockBaseUrl(value: URL)

object WireMockSupport {
  // We have to make the wireMockPort constant per-JVM instead of constant
  // per-WireMockSupport-instance because config values containing it are
  // cached in the GGConfig object
  private lazy val wireMockPort = Port.randomAvailable
}

trait WireMockSupport extends BeforeAndAfterAll with BeforeAndAfterEach {
  me: Suite =>

  val wireMockPort: Int = WireMockSupport.wireMockPort
  val wireMockHost = "localhost"
  val wireMockBaseUrlAsString = s"http://$wireMockHost:$wireMockPort"
  val wireMockBaseUrl = new URL(wireMockBaseUrlAsString)
  protected implicit val implicitWireMockBaseUrl = WireMockBaseUrl(wireMockBaseUrl)

  protected def basicWireMockConfig(): WireMockConfiguration = wireMockConfig()

  private val wireMockServer = new WireMockServer(basicWireMockConfig().port(wireMockPort))

  override protected def beforeAll(): Unit = {
    super.beforeAll()
    configureFor(wireMockHost, wireMockPort)
    wireMockServer.start()
  }

  override protected def afterAll(): Unit = {
    wireMockServer.stop()
    super.afterAll()
  }

  override protected def beforeEach(): Unit = {
    super.beforeEach()
    reset()
  }
} 

//设置package包名称以及导入依赖的类
package bridgeapp.crawler.parsers

import java.net.URL

import akka.actor.{Props, ActorSystem, Actor, ActorRef}
import bridgeapp.crawler.Config
import bridgeapp.crawler.execution.{Response, ResponseParser}
import bridgeapp.crawler.storage.{DiskForumsStorage, ForumsStorage}
import com.typesafe.scalalogging.LazyLogging
import org.jsoup.Jsoup

import scala.collection.JavaConverters._


class MainPageResponseParser(parser: ActorRef) extends ResponseParser {
  override def ->(response: Response): Unit = parser ! response
}

object MainPageResponseParser {

  def apply()(implicit actorSystem: ActorSystem): MainPageResponseParser = {
    val parser = actorSystem.actorOf(Props(new MainPageParser(ForumsStorage())))
    new MainPageResponseParser(parser)
  }
}

class MainPageParser(forumsListStorage: ForumsStorage) extends Actor with LazyLogging {
  override def receive: Receive = {
    case response: Response =>

      val charset = response.charset.getOrElse("utf-8")
      val body = new String(response.body, charset)
      val document = Jsoup.parse(body, response.uri.toString)

      val forumLink = document.select("[href^=viewforum.php]").asScala.toArray

      logger.trace(s" Total url: ${forumLink.length}")

      val forumsIds: Array[Int] = forumLink.map(_.attr("abs:href")).collect {
        case href: String =>
          val s = new URL(href).getQuery.split("&").map { part =>
            val pair = part.split("=")
            pair(0) -> pair(1)
          }.toMap
          s.getOrElse("f", "0").toInt
      }

      logger.trace(s" Extracted forums ids: ${forumsIds.length}")

      forumsListStorage.write(forumsIds, Config.forumsStorageURI)(context.dispatcher)

  }
} 

//设置package包名称以及导入依赖的类
package au.csiro.data61.magda.spatial

import java.net.URL

import au.csiro.data61.magda.AppConfig
import com.typesafe.config.{ Config, ConfigObject }

import scala.collection.JavaConversions._

case class RegionSource(
  name: String,
  url: URL,
  idProperty: String,
  nameProperty: String,
  includeIdInName: Boolean,
  disabled: Boolean,
  order: Int)

object RegionSource {
  def generateRegionId(regionType: String, id: String) = s"${regionType}/$id".toLowerCase
}

class RegionSources(config: Config) {
  val sources = loadFromConfig(config)

  private lazy val lookup = sources.groupBy(_.name.toLowerCase).mapValues(_.head)

  def forName(name: String): Option[RegionSource] = lookup.get(name.toLowerCase)

  private def loadFromConfig(config: Config): Seq[RegionSource] = {
    config.root().map {
      case (name: String, config: ConfigObject) =>
        val regionSourceConfig = config.toConfig()
        RegionSource(
          name = name,
          url = new URL(regionSourceConfig.getString("url")),
          idProperty = regionSourceConfig.getString("idField"),
          nameProperty = regionSourceConfig.getString("nameField"),
          includeIdInName = if (regionSourceConfig.hasPath("includeIdInName")) regionSourceConfig.getBoolean("includeIdInName") else false,
          disabled = regionSourceConfig.hasPath("disabled") && regionSourceConfig.getBoolean("disabled"),
          order = regionSourceConfig.getInt("order"))
    }.toSeq.filterNot(_.disabled)
  }
} 

//设置package包名称以及导入依赖的类
package au.csiro.data61.magda.test.util

import au.csiro.data61.magda.test.util.Generators
import org.scalacheck.Gen
import au.csiro.data61.magda.indexer.external.InterfaceConfig
import java.net.URL
import org.scalacheck.Arbitrary._

object IndexerGenerators {
  val urlGen = for {
    scheme <- Gen.oneOf("http", "https")
    host <- Gen.alphaNumStr
    tld <- Gen.oneOf("com", "net", "com.au", "org", "de")
    path <- Gen.listOf(Gen.alphaNumStr).map(_.mkString("/"))
  } yield new URL(s"$scheme://$host.$tld/$path")

  val interfaceConfInnerGen = for {
    name <- Generators.listSizeBetween(1, 50, Gen.alphaNumChar).map(_.mkString).suchThat(!_.isEmpty)
    interfaceType <- arbitrary[String]
    baseUrl <- urlGen
    pageSize <- Gen.choose(1, 30)
    defaultPublisherName <- Gen.option(arbitrary[String])
  } yield InterfaceConfig(
    name = name,
    interfaceType = interfaceType,
    baseUrl = baseUrl,
    pageSize = pageSize,
    defaultPublisherName = defaultPublisherName
  )

  val interfaceConfsGen = Generators.cachedListGen("interfaceConf", interfaceConfInnerGen, 5)(scala.collection.mutable.HashMap.empty)
  val interfaceConfGen = interfaceConfsGen.flatMap(Gen.oneOf(_))
} 

//设置package包名称以及导入依赖的类
package apodemakeles.scaluty

import java.net.{URI, URL}


  implicit class RichURL(url: URL)
  {
    final lazy val queryMap: Map[String, String] = evaluateQueryMap()

    private def evaluateQueryMap(): Map[String, String] = {
      Option(url.getQuery) match {
        case None => Map.empty[String, String]
        case Some(query) =>
          query
            .split("&")
            .filter(p => p.trim != "" && p.exists(c => c == '='))
            .map
            {
              p =>
                val array = p.split('=')
                if (array.length > 1) (array(0), array(1)) else (array(0), "")
            }.toMap

      }
    }

    def getQueryMap = queryMap

    def getPathWithDefault = if (url.getPath == null || url.getPath == "") "/" else url.getPath

    def appendQuery(tup: (String, String)): URL = {

      var query = if (url.getQuery == null) "" else url.getQuery

      if (query != "") query += "&"

      query += (tup._1 + "=" + tup._2)

      new URI(
        url.getProtocol,
        url.getUserInfo,
        url.getHost,
        url.getPort,
        url.getPath,
        query,
        null).toURL

    }
  }
} 

//设置package包名称以及导入依赖的类
package me.abarrow.ScalaSubNet.mal

import java.io.File
import java.io.FileOutputStream
import java.net.URL
import org.jsoup.Jsoup
import org.jsoup.parser.Parser
import java.nio.channels.Channels

object MALImage {
  def saveMainImage(animeID:Int, imagePath:File):Boolean = {
    val doc = Jsoup.parse(new URL(MALURLs.MAL_ANIME_PAGE_PREFIX + animeID.toString()), 60000)
    val mainImage = doc.select("img.ac").first()
    if (mainImage == null) {
      return false
    }
    val imgSrc = mainImage.attr("src")
    val rbc = Channels.newChannel(new URL(imgSrc).openStream())
    val fos = new FileOutputStream(imagePath)
    try {
      fos.getChannel().transferFrom(rbc, 0, Long.MaxValue)
    } finally {
      fos.close()
      rbc.close()
    }
    true
  }
} 

//设置package包名称以及导入依赖的类
package com.imageintelligence.pix.repository

import java.net.{URL, URLDecoder}
import java.time.Instant

import doobie.imports._

object DoobieHelpers {
  implicit val URLMeta: Meta[URL] =
    Meta[String].nxmap(
      i => new URL(i),
      i => URLDecoder.decode(i.toString, "UTF-8")
    )

  implicit val InstantMeta: Meta[Instant] =
    Meta[java.sql.Timestamp].nxmap(
      i => i.toInstant,
      i => new java.sql.Timestamp(i.toEpochMilli)
    )
} 

//设置package包名称以及导入依赖的类
import java.net.{HttpURLConnection, URL}
import scala.util.Try


  def request(method:String, uri: String, data:String, headers: List[Header]):Try[Response] = {

    Try({
      val obj:URL = new java.net.URL(uri)
      val connection:HttpURLConnection = obj.openConnection().asInstanceOf[HttpURLConnection]
      connection.setRequestMethod(method)

      headers.foreach(item => connection.setRequestProperty(item.property, item.value))

      if (data != null && ("POST".equals(method) || "PUT".equals(method))) {
        connection.setDoOutput(true)
        val dataOutputStream = new java.io.DataOutputStream(connection.getOutputStream())

        dataOutputStream.writeBytes(data)
        dataOutputStream.flush()
        dataOutputStream.close()
      }

      val responseCode = connection.getResponseCode
      val responseMessage = connection.getResponseMessage

      if (isOk(responseCode)) {
        val responseText = new java.util.Scanner(connection.getInputStream, "UTF-8").useDelimiter("\\A").next()
        new Response(responseCode, responseMessage, responseText)
      } else {
        new Response(responseCode, responseMessage, null)
      }
    })
  }
} 

//设置package包名称以及导入依赖的类
package org.argus.jawa.core.classpath

import java.io.File
import java.net.URL

import FileUtils.AbstractFileOps
import org.argus.jawa.core.io.{AbstractFile, FileZipArchive}


trait ZipArchiveFileLookup[FileEntryType <: ClassRepClasspathEntry] extends FlatClasspath {
  val zipFile: File

  assert(zipFile != null, "Zip file in ZipArchiveFileLookup cannot be null")

  override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL)
  override def asClasspathStrings: Seq[String] = Seq(zipFile.getPath)

  private val archive = new FileZipArchive(zipFile)

  override private[jawa] def packages(inPackage: String): Seq[PackageEntry] = {
    val prefix = PackageNameUtils.packagePrefix(inPackage)
    for {
      dirEntry <- findDirEntry(inPackage).toSeq
      entry <- dirEntry.iterator if entry.isPackage
    } yield PackageEntryImpl(prefix + entry.name)
  }

  protected def files(inPackage: String): Seq[FileEntryType] =
    for {
      dirEntry <- findDirEntry(inPackage).toSeq
      entry <- dirEntry.iterator if isRequiredFileType(entry)
    } yield createFileEntry(entry)

  override private[jawa] def list(inPackage: String): FlatClasspathEntries = {
    val foundDirEntry = findDirEntry(inPackage)

    foundDirEntry map { dirEntry =>
      val pkgBuf = collection.mutable.ArrayBuffer.empty[PackageEntry]
      val fileBuf = collection.mutable.ArrayBuffer.empty[FileEntryType]
      val prefix = PackageNameUtils.packagePrefix(inPackage)

      for (entry <- dirEntry.iterator) {
        if (entry.isPackage)
          pkgBuf += PackageEntryImpl(prefix + entry.name)
        else if (isRequiredFileType(entry))
          fileBuf += createFileEntry(entry)
      }
      FlatClasspathEntries(pkgBuf, fileBuf)
    } getOrElse FlatClasspathEntries(Seq.empty, Seq.empty)
  }

  private def findDirEntry(pkg: String) = {
    val dirName = s"${FileUtils.dirPath(pkg)}/"
    archive.allDirs.get(dirName)
  }

  protected def createFileEntry(file: FileZipArchive#Entry): FileEntryType
  protected def isRequiredFileType(file: AbstractFile): Boolean
} 

//设置package包名称以及导入依赖的类
package au.csiro.data61.magda.crawler

import java.net.URL
import au.csiro.data61.magda.external.ExternalInterface
import au.csiro.data61.magda.external.ExternalInterface.ExternalInterfaceType._
import au.csiro.data61.magda.external.InterfaceConfig
import au.csiro.data61.magda.model.misc._

case object ScrapeAll
case object ScrapeRepo
case class ScrapeRepoFinished(baseUrl: URL)
case class ScrapeRepoFailed(baseUrl: URL, reason: Throwable)
case class ScrapeDataSets(start: Long, number: Long)
case class ScrapeDataSetsFinished(start: Long, number: Long)
case class ScrapeDataSetsFailed(start: Long, number: Long, reason: Throwable)
case class Index(source: String, dataSets: List[DataSet])
case object NeedsReIndexing
case class IndexFinished(dataSets: List[DataSet], source: String)
case class IndexFailed(source: String, reason: Throwable) 

//设置package包名称以及导入依赖的类
package au.csiro.data61.magda.external

import com.typesafe.config.Config
import java.net.URL
import au.csiro.data61.magda.external.ExternalInterface.ExternalInterfaceType
import au.csiro.data61.magda.external.ExternalInterface.ExternalInterfaceType._

case class InterfaceConfig(name: String, interfaceType: ExternalInterfaceType, baseUrl: URL, pageSize: Long, fakeConfig: Option[FakeConfig])
case class FakeConfig(datasetCount: Long, datasetPath: String)

object InterfaceConfig {
  def apply(config: Config): InterfaceConfig = {
    val isFaked = config.hasPath("isFaked") && config.getBoolean("isFaked")

    new InterfaceConfig(
      name = config.getString("name"),
      interfaceType = ExternalInterfaceType.withName(config.getString("type")),
      baseUrl = new URL(config.getString("baseUrl")),
      pageSize = config.getLong("pageSize"),
      fakeConfig = {
        if (isFaked && config.hasPath("fake"))
          Some(new FakeConfig(config.getLong("fake.datasetTotal"), config.getString("fake.dataFilePath")))
        else None
      }
    )
  }
} 

//设置package包名称以及导入依赖的类
package pl.mojepanstwo.sap.toakoma.services

import java.net.URL
import java.io.File
import org.apache.commons.io.FileUtils
import org.jsoup.nodes.Document
import com.gargoylesoftware.htmlunit._
import org.jsoup.Jsoup

trait Scraper {
  def get(url: String) : Document
  def dowloadFile(fileUrl:String, filePath:String) : String
}

class DefaultScraperService extends Scraper {

  val webClient = new WebClient

  def get(url: String) : Document = {
      webClient.setRefreshHandler(new RefreshHandler {
        override def handleRefresh(page: Page, url: URL, i: Int): Unit = webClient.getPage(url)
      })
      val apPage: Page = webClient.getPage(url)
      Jsoup.parse(apPage.getWebResponse.getContentAsString)
  }

  def dowloadFile(fileUrl:String, filePath:String) : String = {
    val url = new URL(fileUrl)
    val tmp = new File(filePath)
    FileUtils.copyURLToFile(url, tmp)
    tmp.getAbsolutePath()
  }

} 

//设置package包名称以及导入依赖的类
package la.dp.ingestion3

import java.net.URL

import org.apache.http.client.utils.URIBuilder


  def buildQueryUrl(params: Map[String, String]): URL = {
    // Required properties, not sure if this is the right style
    assume(params.get("endpoint").isDefined)
    assume(params.get("verb").isDefined)
    assume(params.get("metadataPrefix").isDefined)

    val verb = params.get("verb").get
    val metadataPrefix = params.get("metadataPrefix").get
    val url = new URL(params.get("endpoint").get)

    // Build the URL
    val urlParams = new URIBuilder()
      .setScheme(url.getProtocol)
      .setHost(url.getHost)
      .setPath(url.getPath)
      .setParameter("verb", verb)

    params.get("resumptionToken") match {
      case Some(v) if v.nonEmpty => urlParams.setParameter("resumptionToken", v)
      case _ => urlParams.setParameter("metadataPrefix", metadataPrefix)
    }

    urlParams.build.toURL
  }
}

trait QueryUrlBuilder {
  def buildQueryUrl (params: Map[String, String]): URL

} 

//设置package包名称以及导入依赖的类
package mesosphere.mesos.scale

import java.net.URL

import mesosphere.mesos.scale.MetricsFormat._

import scala.collection.immutable.Seq


  def main(args: Array[String]): Unit = {
    def printSlope(metrics: Map[Metric, Metric]): Unit = {
      import DisplayHelpers._
      val header = Vector("Metric", "Base", "Sample", "Increase in %")
      val rows: Seq[IndexedSeq[String]] = metrics.map {
        case (a, b) => IndexedSeq(a.name, a.mean, b.mean, (b.mean / a.mean * 100).toInt - 100).map(_.toString)
      }(collection.breakOut)
      printTable(Seq(left, right, right, right), withUnderline(header) ++ rows)
    }

    if (args.length == 3) {
      println("\n\nMetrics that got worse (deterioration factor == 1):")
      printSlope(filterDeteriorated(new URL(args(0)), new URL(args(1)), 1))
      println(s"\n\nMetrics that got deteriorated (deterioration factor == ${args(2)}):")
      val deteriorated = filterDeteriorated(new URL(args(0)), new URL(args(1)), args(2).toDouble)
      if (deteriorated.nonEmpty) {
        printSlope(deteriorated)
        throw new IllegalStateException(s"Sample is deteriorated according to deterioration factor")
      }
    } else {
      println(
        """Usage:
          | FindDeterioratedMetrics <file_base> <file_sample> <deterioration_factor>"
          | file_base: the file with the base metrics
          | file_sample: the file with the actual sampled metrics
          | deterioration_factor: number [0..] where 1 means the same as the base line
        """.stripMargin)
      sys.exit(1)
    }
  }
}