node.js - Unzipping files in Lambda returns null, but lambda-local has the expected response

Question

Firstly, thanks for any help you can give me. I've been struggling with this code for an embarassingly long time and I can't figure it out. This lambda function is written in NodeJS and is used to retrieve a file from S3, unzip it and extract headers from each of the files contained in the zip. Instead, it returns null on Lambda and stalls locally.

Expected Behaviour

1. A zip file is retrieved from S3 using the information contained in the event
2. The zip file is unzipped and stored in a files array
3. Cycle through this array in batches of 10, creating an extraction promise for each file and waiting for each batch of 10 promises to resolve before moving on to the next
4. When all promises have resolved, return a success response

Actual Behaviour: Lambda(As far as I can tell)

1. A zip file is retrieved from S3 using the information contained in the event
2. The zip file is unzipped and stored in a files array
3. Cycle through this array in batches of 10, creating an extraction promise for each file and waiting for each batch of 10 promises to resolve before moving on to the next
4. Return null without exceeding allocated memory or timing out

Actual Behaviour: Lambda-local(As far as I can tell)

1. A zip file is retrieved from S3 using the information contained in the event
2. The zip file is unzipped and stored in a files array
3. Cycle through this array in batches of 10, creating an extraction promise for each file and waiting for each batch of 10 promises to resolve before moving on to the next
4. Finishes the first 46-50 promises, then lags for several minutes, finally completing.

Event

{
  "Records": [
      "s3": {
        "bucket": {
          "name": "randombucket"
        },
        "object": {
          "key": "a/b/file.zip"
        }
      }
    }
  ]
}

index.js

const AWS = require('aws-sdk');
const dicomParser = require('dicom-parser');
const unzipper = require('unzipper');
const { dicomTags } = require('./dicomTags');

AWS.config.update({ region: process.env.region });
const s3 = new AWS.S3({ apiVersion: '2006-03-01'});
const lambda = new AWS.Lambda();

const transformByteTagsToEnglish = async (byteDicomHeader) => {
  const readableDicomHeader = new Promise((resolve, reject) => {
    const newHeader = {};
    try {
      Object.keys(byteDicomHeader).map((key) => {
        newHeader[dicomTags[key]] = byteDicomHeader[key];
      });
      resolve(newHeader);
    } catch(error) {
      reject(`Problem transforming byte tags into a readable format ${error}`);
    }
  });

  return readableDicomHeader;
}

const extractDicomHeader = async (file) => {
  return await file.buffer()
  .then((byteBuffer) => new Uint8Array(byteBuffer))
  .then((byteArray) => dicomParser.parseDicom(byteArray))
  .then((wholeDicom) => dicomParser.explicitDataSetToJS(wholeDicom))
  .then((headerObjectByteTags) => transformByteTagsToEnglish(headerObjectByteTags))
  .catch((error) => {
    console.log("Dicom Header Extraction Error", error);
  });
}

exports.handler = async function(event) {
  try {
    const bucket = event.Records[0].s3.bucket.name;
    const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/+/g, ' '));

    const directory = await unzipper.Open.s3(s3, {Bucket: bucket, Key: key});
    const { files } = directory;

    const fileBatchSize = 10;
    for (let i = 0; i < files.length; i+=Math.min(fileBatchSize, files.length - i)) {
      const fileProcessingPromises = [];

      for (let j = 0; j < Math.min(fileBatchSize, files.length - i); j++) {
        const targetFileIndex = i + j;
        fileProcessingPromises.push(extractDicomHeader(files[targetFileIndex]));
      }

      await Promise.all(fileProcessingPromises)
        .catch((error) => {
          throw new Error(error);
        });
    }
    
    const successApiResponse = {
      status: 200,
      message: 'Success!'
    };

    return successApiResponse;
  } catch (error) {
    const errorApiResponse = {
      status: 500,
      message: 'Failure :(',
      error
    };

    return errorApiResponse;
  }
}

question from:https://stackoverflow.com/questions/65920803/unzipping-files-in-lambda-returns-null-but-lambda-local-has-the-expected-respon

Answer 1

Waitting for answers