javascript - How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js

Question

How do I copy all objects from one prefix to other? I have tried all possible ways to copy all objects in one shot from one prefix to other, but the only way that seems to work is by looping over a list of objects and copying them one by one. This is really inefficient. If I have hundreds of files in a folder, will I have to make 100 calls?

var params = {
         Bucket: bucket,
         CopySource: bucket+'/'+oldDirName+'/filename.txt',
         Key: newDirName+'/filename.txt',
 };
s3.copyObject(params, function(err, data) {
  if (err) {
      callback.apply(this, [{
          type: "error",
          message: "Error while renaming Directory",
          data: err
      }]);
  } else {
      callback.apply(this, [{
          type: "success",
          message: "Directory renamed successfully",
          data: data
      }]);
  }
});

question from:https://stackoverflow.com/questions/30959251/how-to-copy-move-all-objects-in-amazon-s3-from-one-prefix-to-other-using-the-aws

Answer 1

You will need to make one AWS.S3.listObjects() to list your objects with a specific prefix. But you are correct in that you will need to make one call for every object that you want to copy from one bucket/prefix to the same or another bucket/prefix.

You can also use a utility library like async to manage your requests.

var AWS = require('aws-sdk');
var async = require('async');
var bucketName = 'foo';
var oldPrefix = 'abc/';
var newPrefix = 'xyz/';
var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});

var done = function(err, data) {
  if (err) console.log(err);
  else console.log(data);
};

s3.listObjects({Prefix: oldPrefix}, function(err, data) {
  if (data.Contents.length) {
    async.each(data.Contents, function(file, cb) {
      var params = {
        Bucket: bucketName,
        CopySource: bucketName + '/' + file.Key,
        Key: file.Key.replace(oldPrefix, newPrefix)
      };
      s3.copyObject(params, function(copyErr, copyData){
        if (copyErr) {
          console.log(copyErr);
        }
        else {
          console.log('Copied: ', params.Key);
          cb();
        }
      });
    }, done);
  }
});

Hope this helps!