Better import for very large GTFS file

Increase Mongo timeout on import.js, version bump
BlinkTagInc · May 3, 2017 · d863fb3 · d863fb3
1 parent dcc6e73
commit d863fb3
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/lib/import.js b/lib/import.js
@@ -74,7 +74,7 @@ function removeDatabase(task, cb) {
     return cb(null, task);
   }
 
-  return async.forEach(filenames, (filename, cb) => {
+  return async.forEachSeries(filenames, (filename, cb) => {
     task.db.collection(filename.collection, (err, collection) => {
       if (err) {
         return cb(err);
@@ -350,7 +350,13 @@ module.exports = (config, cb) => {
   log(`Starting GTFS import for ${fileCount} ${utils.pluralize('file', fileCount)}`);
 
   // Open database and create queue for agency list
-  MongoClient.connect(config.mongoUrl, {w: 1}).then(db => {
+  const mongoOptions = {
+    w: 1,
+    connectTimeoutMS: 120000,
+    socketTimeoutMS: 120000
+  };
+
+  MongoClient.connect(config.mongoUrl, mongoOptions).then(db => {
     const q = async.queue(importGTFS, 1);
     // Loop through all agencies specified
     config.agencies.forEach(item => {

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "gtfs",
   "description": "Import GTFS transit data into mongodb and query routes, stops, times, fares and more",
-  "version": "0.9.9",
+  "version": "0.9.10",
   "keywords": [
     "transit",
     "gtfs",