import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * Test IPT bug 2158
 *
 * @see <a href="http://code.google.com/p/gbif-providertoolkit/source/detail?r=2158">IPT revision 2158</a>
 */
@Test
public void testIssue2158() throws UnsupportedArchiveException, IOException {
  // test zip with 1 extension file
  File zip = FileUtils.getClasspathFile("archive-tax.zip");
  File tmpDir = Files.createTempDirectory("dwca-io-test").toFile();
  CompressionUtil.decompressFile(tmpDir, zip);
  // read archive from this tmp dir
  Archive arch = ArchiveFactory.openArchive(tmpDir);
  assertNotNull(arch.getCore().getId());
  assertEquals(1, arch.getExtensions().size());

  boolean found = false;
  for (Record rec : arch.getCore()) {
    if ("113775".equals(rec.id())) {
      found = true;
      assertEquals(
        "Ehrenberg, 1832, in Hemprich and Ehrenberg, Symbolæ Phisicæ Mammalia, 2: ftn. 1 (last page of fascicle headed \"Herpestes leucurus H. E.\").",
        rec.value(DwcTerm.originalNameUsageID));
    }
  }
  assertTrue(found);
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * The pensoft archive http://pensoft.net/dwc/bdj/checklist_980.zip
 * contains empty extension files which caused NPE in the dwca reader.
 */
@Test
public void testExtensionNPE() throws UnsupportedArchiveException, IOException {
  File zip = FileUtils.getClasspathFile("checklist_980.zip");
  File tmpDir = Files.createTempDirectory("dwca-io-test").toFile();
  CompressionUtil.decompressFile(tmpDir, zip);
  // read archive from this tmp dir
  Archive arch = ArchiveFactory.openArchive(tmpDir);
  assertNotNull(arch.getCore().getId());
  assertEquals(3, arch.getExtensions().size());

  boolean found = false;
  for (StarRecord rec : arch) {
    if ("980-sp10".equals(rec.core().id())) {
      found = true;
    }
  }
  assertTrue(found);
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * Constructor from a URI
 *
 * @param source
 * @throws IOException
 */
public DwcArchive(URL source) throws IOException {

    File dwca = File.createTempFile("dwcaFile-", null);
    dwca.deleteOnExit();

    System.out.println("Downloading DarwinCore archive " + source.toString());
    try {
        // download
        DefaultHttpClient client = new DefaultHttpClient();
        HttpUtil http = new HttpUtil(client);
        http.download(source, dwca);
    } catch (Exception e) {
        System.out.println("Cannot download DarwinCore archive. Abort.");
        System.exit(0);
    }

    location = CompressionUtil.decompressFile(dwca);
    System.out.println("Uncompressed into directory: " + location.getAbsolutePath());
    archive = ArchiveFactory.openArchive(location);
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * Opens an archive from a local file and decompresses or copies it into the given archive directory.
 * Make sure the archive directory does not contain files already!
 *
 * @param archiveFile the location of a compressed archive or single data file
 * @param archiveDir  empty, writable directory used to keep decompress archive in
 */
public static Archive openArchive(File archiveFile, File archiveDir) throws IOException, UnsupportedArchiveException {
	// try to decompress archive
	try {
		List<File> files = CompressionUtil.decompressFile(archiveDir, archiveFile);
		// continue to read archive from the tmp dir
		return openArchive(archiveDir);

	} catch (CompressionUtil.UnsupportedCompressionType e) {
		// If its a text file only we will get this exception - but also for corrupt compressions
		// try to open as text file only
		return openArchive(archiveFile);
	}
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
static Archive fromCompressed(Path dwcaLocation, Path destination) throws IOException, UnsupportedArchiveException {
  if (!Files.exists(dwcaLocation)) {
    throw new FileNotFoundException("dwcaLocation does not exist: " + dwcaLocation.toAbsolutePath());
  }

  if (Files.exists(destination)) {
    // clean up any existing folder
    LOG.debug("Deleting existing archive folder [{}]", destination.toAbsolutePath());
    org.gbif.utils.file.FileUtils.deleteDirectoryRecursively(destination.toFile());
  }
  FileUtils.forceMkdir(destination.toFile());
  // try to decompress archive
  try {
    CompressionUtil.decompressFile(destination.toFile(), dwcaLocation.toFile(), true);
    // we keep subfolder, but often the entire archive is within one subfolder. Remove that root folder if present
    File[] rootFiles = destination.toFile().listFiles((FileFilter) HiddenFileFilter.VISIBLE);
    if (rootFiles.length == 1) {
      File root = rootFiles[0];
      if (root.isDirectory()) {
        // single root dir, flatten structure
        LOG.debug("Removing single root folder {} found in decompressed archive", root.getAbsoluteFile());
        for (File f : FileUtils.listFiles(root, TrueFileFilter.TRUE, null)) {
          File f2 = new File(destination.toFile(), f.getName());
          f.renameTo(f2);
        }
      }
    }
    // continue to read archive from the tmp dir
    return fromLocation(destination);
  } catch (CompressionUtil.UnsupportedCompressionType e) {
    throw new UnsupportedArchiveException(e);
  }
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * Test GNUB style dwca with a single tab delimited file that has a .tab suffix.
 */
@Test
public void testGnubTabZip() throws UnsupportedArchiveException, IOException {
  // test GNUB zip with 1 data file
  File tmpDir = Files.createTempDirectory("dwca-io-test").toFile();
  tmpDir.deleteOnExit();
  File zip = FileUtils.getClasspathFile("gnub.tab.zip");
  CompressionUtil.decompressFile(tmpDir, zip);

  // read archive from this tmp dir
  Archive arch = ArchiveFactory.openArchive(tmpDir);

  Record rec = arch.getCore().iterator().next();
  assertEquals("246daa62-6fce-448f-88b4-94b0ccc89cf1", rec.id());
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
private void download(String url) throws IOException {
  HttpUtil hutil = new HttpUtil(HttpUtil.newMultithreadedClient(2000, 5, 2));

  File dwca = File.createTempFile("clb", ".dwca");
  hutil.download(url, dwca);
  System.out.println("Downloaded raw archive to " + dwca.getAbsolutePath());
  CompressionUtil.decompressFile(nCfg.archiveDir(datasetKey), dwca);
  System.out.println("Decompressed archive to " + nCfg.archiveDir(datasetKey));
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
@Test
public void manyNormalizersInParallel() throws Exception {
  final int tasks = 500;

  ExecutorCompletionService<Object> ecs = new ExecutorCompletionService(Executors.newFixedThreadPool(threads));
  List<Future<Object>> futures = Lists.newArrayList();
  for (int i = 0; i < tasks; i++) {
    UUID dk = UUID.randomUUID();

    // copy dwca
    File dwca = cfgN.archiveDir(dk);
    CompressionUtil.decompressFile(dwca, this.zip);

    Normalizer normalizer = Normalizer.create(cfgN, registry, dk);
    System.out.println("Submit normalizer " + i);
    futures.add(ecs.submit(Executors.callable(normalizer)));
  }

  int idx = 1;
  for (Future<Object> f : futures) {
    f.get();
    System.out.println("Finished normalizer " + idx++);
    monitor.run();
  }
  System.out.println("Finished all jobs");
  monitor.run();
  for (Thread t : Thread.getAllStackTraces().keySet()) {
    System.out.println(t.getState() + " " + t.getName());
  }
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
@Test
public void manyNormalizersInParallel() throws Exception {
    final int tasks = 500;

    NormalizerConfiguration cfgN = new NormalizerConfiguration();
    cfgN.neo.neoRepository = Files.createTempDirectory("neotest").toFile();
    cfgN.archiveRepository = Files.createTempDirectory("neotestdwca").toFile();

    MetricRegistry registry = new MetricRegistry();

    File zip = new File(getClass().getResource("/plazi.zip").getFile());
    zip = new File("/Users/markus/code/checklistbank/checklistbank-cli/src/test/resources/plazi.zip");

    Timer timer = new Timer();
    ResourcesMonitor monitor = new ResourcesMonitor();
    timer.schedule(monitor, 500);


    ExecutorCompletionService<InsertMetadata> ecs = new ExecutorCompletionService(Executors.newFixedThreadPool(threads));
    List<Future<InsertMetadata>> futures = Lists.newArrayList();
    for (int i = 0; i < tasks; i++) {
        UUID dk = UUID.randomUUID();

        // copy dwca
        File dwca = cfgN.archiveDir(dk);
        CompressionUtil.decompressFile(dwca, zip);

        System.out.println("Submit inserter job " + i);
        futures.add(ecs.submit(new InserterJob(dk, cfgN, registry)));
    }

    int idx = 1;
    for (Future<InsertMetadata> f : futures) {
        f.get();
        System.out.println("Finished inserter " + idx++);
        monitor.run();
    }
    System.out.println("Finished all jobs");
    System.out.println("Open files: " + monitor.getOpenFileDescriptorCount());
    System.out.println("Running threads: " + Thread.getAllStackTraces().size());
    for (Thread t : Thread.getAllStackTraces().keySet()) {
        System.out.println(t.getState() + " " + t.getName());
    }

    System.out.println("Cleaning up artifacts...");
    FileUtils.deleteQuietly(cfgN.neo.neoRepository);
    FileUtils.deleteQuietly(cfgN.archiveRepository);
}
 

import org.gbif.utils.file.CompressionUtil; //导入依赖的package包/类
/**
 * Main method to assemble the dwc archive and do all the work until we have a final zip file.
 *
 * @param zipFile the final zip file holding the entire archive
 */
public void buildArchive(File zipFile) throws DownloadException {
  LOG.info("Start building the archive {} ", zipFile.getPath());

  try {
    if (zipFile.exists()) {
      zipFile.delete();
    }
    if (!configuration.isSmallDownload()) {
      // oozie might try several times to run this job, so make sure our filesystem is clean
      cleanupFS();

      // create the temp archive dir
      archiveDir.mkdirs();
    }

    // metadata, citation and rights
    License downloadLicense = addConstituentMetadata();

    // persist the License assigned to the download
    persistDownloadLicense(configuration.getDownloadKey(), downloadLicense);

    // metadata about the entire archive data
    generateMetadata();

    // meta.xml
    DwcArchiveUtils.createArchiveDescriptor(archiveDir);

    // zip up
    LOG.info("Zipping archive {}", archiveDir.toString());
    CompressionUtil.zipDir(archiveDir, zipFile, true);

    // add the large download data files to the zip stream
    if (!configuration.isSmallDownload()) {
      appendPreCompressedFiles(zipFile);
    }
    targetFs.moveFromLocalFile(new Path(zipFile.getPath()),
                               new Path(workflowConfiguration.getHdfsOutputPath(), zipFile.getName()));

  } catch (IOException e) {
    throw new DownloadException(e);

  } finally {
    // always cleanUp temp dir
    cleanupFS();
  }

}