node.js - Inserting a big array of object in mongodb from nodejs

Question

I need to insert a big array of objects (about 1.5-2 millions) in mongodb from nodejs. How can i improve my inserting?

This is my code:

var sizeOfArray = arrayOfObjects.length; //sizeOfArray about 1.5-2 millions
for(var i = 0; i < sizeOfResult; ++i) {
  newKey = {
    field_1: result[i][1],
    field_2: result[i][2],
    field_3: result[i][3]
  };
  collection.insert(newKey, function(err, data) {
    if (err) {
      log.error('Error insert: ' + err);
    }
  });
}

Answer 1

You can use bulk inserts.

There are two types of bulk operations:

1. Ordered bulk operations. These operations execute all the operation in order and error out on the first write error.
2. Unordered bulk operations. These operations execute all the operations in parallel and aggregates up all the errors. Unordered bulk operations do not guarantee order of execution.

So you can do something like this:

var MongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://myserver:27017/test", function(err, db) {
    // Get the collection
    var col = db.collection('myColl');

    // Initialize the Ordered Batch
    // You can use initializeUnorderedBulkOp to initialize Unordered Batch
    var batch = col.initializeOrderedBulkOp();

    for (var i = 0; i < sizeOfResult; ++i) {
      var newKey = {
          field_1: result[i][1],
          field_2: result[i][2],
          field_3: result[i][3]
      };
      batch.insert(newKey);
    }

    // Execute the operations
    batch.execute(function(err, result) {
      console.dir(err);
      console.dir(result);
      db.close();
    });
});