Question

因此，我跟随本教程学习了亚马逊https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html，以便为我上传到S3存储桶的每个图像创建一个缩略图生成器。它可以在500KB左右的小图像上正常工作，并且可以将它们正确地放入缩略图存储桶中，但是我尝试将300MB的图像文件上传到我的S3存储桶中，而lambda函数似乎无法正常工作。

我浏览了其他论坛，并尝试在AWS Lambda中尝试一些超时和内存大小设置，因为我认为该功能可能需要更多的内存，但事实并非如此，我个人也不知道否则我就离开了。

这是直接来自我使用的链接的lambda函数的副本，当设置了MAX_HEIGHT和MAX_WIDTH时，错误发生在第57行。大文件的大小似乎总是不确定的。

// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var gm = require('gm')
            .subClass({ imageMagick: true }); // Enable ImageMagick integration.
var util = require('util');

// constants
var MAX_WIDTH  = 100;
var MAX_HEIGHT = 100;

// get reference to S3 client 
var s3 = new AWS.S3();

exports.handler = function(event, context, callback) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    // Object key may have spaces or unicode non-ASCII characters.
    var srcKey    =
    decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));  
    var dstBucket = srcBucket + "resized";
    var dstKey    = "resized-" + srcKey;

    // Sanity check: validate that source and destination are different buckets.
    if (srcBucket == dstBucket) {
        callback("Source and destination buckets are the same.");
        return;
    }

    // Infer the image type.
    var typeMatch = srcKey.match(/\.([^.]*)$/);
    if (!typeMatch) {
        callback("Could not determine the image type.");
        return;
    }
    var imageType = typeMatch[1];
    if (imageType != "jpg" && imageType != "png") {
        callback('Unsupported image type: ${imageType}');
        return;
    }

    // Download the image from S3, transform, and upload to a different S3 bucket.
    async.waterfall([
        function download(next) {
            // Download the image from S3 into a buffer.
            s3.getObject({
                    Bucket: srcBucket,
                    Key: srcKey
                },
                next);
            },
        function transform(response, next) {
            gm(response.Body).size(function(err, size) {
                // Infer the scaling factor to avoid stretching the image unnaturally.
                var scalingFactor = Math.min(
                    MAX_WIDTH / size.width,
                    MAX_HEIGHT / size.height
                );
                var width  = scalingFactor * size.width;
                var height = scalingFactor * size.height;

                // Transform the image buffer in memory.
                this.resize(width, height)
                    .toBuffer(imageType, function(err, buffer) {
                        if (err) {
                            next(err);
                        } else {
                            next(null, response.ContentType, buffer);
                        }
                    });
            });
        },
        function upload(contentType, data, next) {
            // Stream the transformed image to a different S3 bucket.
            s3.putObject({
                    Bucket: dstBucket,
                    Key: dstKey,
                    Body: data,
                    ContentType: contentType
                },
                next);
            }
        ], function (err) {
            if (err) {
                console.error(
                    'Unable to resize ' + srcBucket + '/' + srcKey +
                    ' and upload to ' + dstBucket + '/' + dstKey +
                    ' due to an error: ' + err
                );
            } else {
                console.log(
                    'Successfully resized ' + srcBucket + '/' + srcKey +
                    ' and uploaded to ' + dstBucket + '/' + dstKey
                );
            }

            callback(null, "message");
        }
    );
};

以下是直接来自AWS的Cloud Watch日志的错误消息：

2019-05-14T22:31:28.731Z    b5fccf54-e55f-49f2-9206-462fa5769149    TypeError: Cannot read property 'width' of undefined
at gm.<anonymous> (/var/task/index.js:57:38)
at emitMany (events.js:147:13)
at gm.emit (events.js:224:7)
at gm.<anonymous> (/var/task/node_modules/gm/lib/getters.js:70:16)
at cb (/var/task/node_modules/gm/lib/command.js:322:16)
at gm._spawn (/var/task/node_modules/gm/lib/command.js:226:14)
at gm._exec (/var/task/node_modules/gm/lib/command.js:190:17)
at gm.proto.(anonymous function) [as size] (/var/task/node_modules/gm/lib/getters.js:68:12)
at transform (/var/task/index.js:54:31)
at nextTask (/var/task/node_modules/async/dist/async.js:5324:14)

编辑：此外，我似乎也收到了在两个错误之间交替出现的错误：

2019-05-14T22:55:25.923Z    7a7f1ec2-cd78-4fa5-a296-fee58033aea6    Unable to resize MYBUCKET/image.png and upload to MYBUCKETRESIZE/resize-image.png due to an error: Error: Stream yields empty buffer

编辑：添加了报告行：

REPORT RequestId: a67e1e79-ebec-4b17-9832-4049ff31bd89  Duration: 7164.64 ms    Billed Duration: 7200 ms Memory Size: 1024 MB   Max Memory Used: 810 MB

Answer 1

您的两个错误的原因略有不同。

Stream yields empty buffer几乎肯定意味着您内存不足。

另一个错误可能是特定的imagemagick错误，我认为您没有处理。虽然这也可能与内存问题有关。因为它在回调中，所以您需要检查err变量：

function transform(response, next) {
    gm(response.Body).size(function(err, size) {
        if (err) throw err;

imagemamgick模块可能正在向/ tmp写一些东西。用于检查/清除我从事的项目中的磁盘使用情况的代码段

const fs = require('fs')
const folder = '/tmp/'

// Deleting any files in the /tmp folder (lambda will reuse the same container, we will run out of space)
const files = fs.readdirSync(folder)
for (const file of files) {
  console.log('Deleting file from previous invocation: ' + folder + file)
  fs.unlinkSync(folder + file)
}

AWS Lambda缩略图生成器不适用于较大的文件大小

1 个答案: