node.js - NodeJS Yauzl - extract a subfolder into a new zip

Question

I have a specific usecase that I could not find an example for and trying to hack together different examples is not achieving much for me so far.

I am using a nodeJS function in Lambda on AWS

I have a zipfile in the form of a buffer (just read from S3 using getObject). The zips are never going to exceed 10mb so it's ok to do it in memory and can avoid using streams and hopefully avoid using local tmp.

Each zip has some files and folders, i want to get all files that are in a folder called "src/". This is the same for all zips. The files don't need to be decompressed, i want to put them straight into a new zip without the "src" subfolder - so just the files and folders within "src" on the root of a new zip, compressed, any folders within src/ should retain their hierarchy.

The final result of this should be a buffer of the output zip.

any help would be much appreciated.

question from:https://stackoverflow.com/questions/65829425/nodejs-yauzl-extract-a-subfolder-into-a-new-zip

Answer 1

I may have solved it myself, but i am still open to any improvements on this as it feels a bit hacky and inefficient as it is decompressing then compressing again.

const yauzl = require("yauzl");
const yazl = require("yazl");

const uploadStream = ({ Bucket, Key }) => {
    const s3 = new AWS.S3();
    const pass = new stream.PassThrough();
    return {
      writeStream: pass,
      promise: s3.upload({ Bucket, Key, Body: pass }).promise(),
    };
  }


  // retrieve the artifact
  return s3.getObject({Bucket: bucket, Key: key}).promise().then(res => {

    // unzip the artifact
    return new Promise(function(resolve, reject){

      // new zip that will contain only src
      const newzip = new yazl.ZipFile();

      // read the zip from buffer (entire zip, this cannot be streamed)
      yauzl.fromBuffer(res.Body, {lazyEntries: true}, (err, zip) => {
        if(err) {
          console.log("Error accessing artifact: ",err);
          return reject("Error accessing artifact");
        }

        // read each item in the zip
        zip.readEntry();
        zip.on("entry", function(entry){
          // we only want files in the src dir, skip others
          if(entry.fileName.substr(0,3) !== "src") return zip.readEntry();

          // extract file
          zip.openReadStream(entry, {decompress: entry.isCompressed() ? true : null}, function (err, readStream) {
            if(err){
              zip.close();
              console.log("Failed to read file in artifact: ", err);
              return reject("Failed to read file in artifact");
            }

            // collect data into buffer
            let buffer = null;
            readStream.on('data', function(d) {
              if(!buffer){
                buffer = d;
              }else{
                buffer = Buffer.concat([buffer, d]);
              }
            });

            // file data collection completed
            readStream.on('end', function () {

              // add it to the new zip (without the src dir in the path)
              newzip.addBuffer(buffer,entry.fileName.substr(4));

              // continue to next entry
              zip.readEntry();
            });

            // fail on error
            readStream.on('error', function (err) {
              zip.close();
              console.log("Failed to extract file from artifact: ", err);
              return reject("Failed to extract file from artifact");
            });
          });

        });

        // all items processed
        zip.on("end", function(){
          console.log("Completed extracting all files");

          // all files added
          newzip.end();

          // store on s3
          const { writeStream, promise } = uploadStream({Bucket: bucket, Key: key+"Deploy"});
          newzip.outputStream.pipe(writeStream).on("close", function(){
            resolve({result:true,artifact:key+"Deploy"});
          });

        });

      });

    }).catch(err => {
      console.log("Unzip artifact error: ",err);
      return Promise.reject("Could not unzip artifact");
    });


  }).catch(err => {
    console.log("Retrieve artifact error: ",err);
    return Promise.reject("Could not retrieve artifact");
  });