追加/插入10每5分钟到MongoDB中使用的NodeJS

Question

目前我正在试图插入每5分钟，走出1000数据，我现在对我的csv文件的10个新的数据1000的数据。

我该怎么做？帮助将不胜感激

以下是我对appendMongo.js

var fs= require('fs'); 
 
var csv = require('fast-csv'); 
 

 
var mongodb = require('mongodb'); 
 
var url = 'mongodb://localhost:27017/insertDB'; 
 
var MongoClient = mongodb.MongoClient; 
 

 

 
var data; 
 

 
MongoClient.connect(url, function (err, db) { 
 
    if (err) { 
 
     console.log('Unable to connect to the mongoDB server. Error:', err); 
 
    } else { 
 
     console.log('Connection established to', url); 
 
     var collection = db.collection('airports'); 
 
     readData=fs.createReadStream('1000data.csv').pipe(csv()) 
 
        .on('data',function(data){ 
 
         collection.insert({'data': data}); 
 
        }) 
 
        .on('end',function(data){ 
 
         console.log('Read finished'); 
 
        }) 
 

 
    } 
 
    }); //End of Mongo connect

一切CSV文件的代码是相似的，正如我刚才复制并粘贴文件，因为我需要做的1000数据我被要求这样做。

这是数据出1000个数据的一部分1000data.csv

Machine Unit,Air Temperature °C,Water Temperature °C,Heat Temperature °C,Room Temperature °C,Date,Time 
 
1,61,54,87,20,12/3/2016,8:39AM 
 
2,41,57,92,21,29/9/2016,3:51PM 
 
3,39,53,89,22,22/12/2016,5:30PM 
 
4,56,27,87,23,9/7/2016,6:54AM 
 
5,71,85,76,24,21/10/2016,7:29PM 
 
6,82,64,85,25,22/3/2016,9:38PM 
 
7,91,73,94,26,21/10/2016,1:30PM 
 
8,35,79,89,27,30/4/2016,2:17PM 
 
9,29,59,83,28,18/7/2016,7:49AM 
 
10,19,61,80,29,27/9/2016,8:21PM

Answer 1

看看下面的代码可以帮助你。

var fs= require('fs'); 
 
var csv = require('fast-csv'); 
 

 
var mongodb = require('mongodb'); 
 
var url = 'mongodb://localhost:27017/insertDB'; 
 
var MongoClient = mongodb.MongoClient; 
 
var data; 
 

 
MongoClient.connect(url, function (err, db) { 
 
    if (err) { 
 
     console.log('Unable to connect to the mongoDB server. Error:', err); 
 
    } else { 
 
     console.log('Connection established to', url); 
 
     var collection = db.collection('airports'); 
 
     readData=fs.createReadStream('1000data.csv').pipe(csv()) 
 
        .on('data',function(data) 
 
        { 
 
         var count = 0; // Count initialised to 0 
 
         var timeout = setInterval(function() // Insert 10 record after 5 minutes 
 
         { 
 
         if(count >= 999) // If count is 1000 stop inserting. 
 
         { 
 
          clearInterval(timeout); 
 
         } 
 
         else 
 
         { 
 
          for(var i = count; i < count+10; i++) 
 
          { 
 
          collection.insert({'data': data[i]}, 
 
          function(err) 
 
          { 
 
           console.log(err); 
 
          }); //Insert one record 
 
          } 
 
          count += 10; //Increase the count 
 
         
 
         } 
 
         },300000); 
 
        }) 
 
        .on('end',function(data){ 
 
         console.log('Read finished'); 
 
        }) 
 

 
    } 
 
    }); //End of Mongo connect

或者你可以尝试下面的代码

var fs= require('fs'); 
 
var csv = require('fast-csv'); 
 

 
var mongodb = require('mongodb'); 
 
var url = 'mongodb://localhost:27017/insertDB'; 
 
var MongoClient = mongodb.MongoClient; 
 
var data; 
 

 
MongoClient.connect(url, function (err, db) { 
 
    if (err) { 
 
     console.log('Unable to connect to the mongoDB server. Error:', err); 
 
    } else { 
 
     console.log('Connection established to', url); 
 
     var collection = db.collection('airports'); 
 
     readData=fs.createReadStream('1000data.csv').pipe(csv()) 
 
        .on('data',function(data) 
 
        { 
 
         var count = 0; // Count initialised to 0 
 
         var timeout = setInterval(function() // Insert 10 record after 5 minutes 
 
         { 
 
         if(count >= 999) // If count is 1000 stop inserting. 
 
         { 
 
          clearInterval(timeout); 
 
         } 
 
         else 
 
         { 
 
          var temp = []; 
 
          for(var i = count; i < count+10; i++) 
 
          { 
 
          temp.push({'data':data[i]}); 
 
          } 
 
          count += 10; //Increase the count 
 
          collection.insert(temp, function(err){ 
 
           console.log(err); 
 
          }); //Insert 10 records at one time. 
 
         } 
 
         },300000); 
 
        }) 
 
        .on('end',function(data){ 
 
         console.log('Read finished'); 
 
        }) 
 

 
    } 
 
    }); //End of Mongo connect

输出你会得到你的数据库将在以下格式：

{data:1,61,54,87,20,12/3/2016,8:39AM}, 
{data:2,41,57,92,21,29/9/2016,3:51PM}, 
{data:3,39,53,89,22,22/12/2016,5:30PM}, 
{data:4,56,27,87,23,9/7/2016,6:54AM}, 
{data:5,71,85,76,24,21/10/2016,7:29PM}, 
{data:6,82,64,85,25,22/3/2016,9:38PM}, 
{data:7,91 ,73,94,26,21/10/2016,1:30PM}, 
{data:8,35,79,89,27,30/4/2016,2:17PM}, 
{data:9,29,59,83,28,18/7/2016,7:49AM}, 
{data:10,19,61,80,29,27/9/2016,8:21PM} 

Answer 2

只需向数组中添加数据并每隔5分钟执行一次就可以执行一个单独的函数。这个函数可以处理mongodb连接工作。

准系统下面

'use strict'; 
var fs= require('fs'); 
var csv = require('fast-csv'); 
var dataArray = []; 
var finishedReading = false; 
//set interval limit as you like 
var interval = 1000 * 10; //10secs 
fs.createReadStream('data.csv').pipe(csv()) 
        .on('data',function(data){ 
         console.log(data); 
         dataArray.push(data) 
        }) 
        .on('end',function(data){ 
         console.log('Read finished'); 
         finishedReading = true; 
        }) 


var intervalFn = setInterval(function(){ 
    //you can have check for 10 since you want that many items 
    if(dataArray.length >= 2){ 
     var twoItems = dataArray.splice(0,2); 
     console.log(new Date() + twoItems); 
     //execute mongodb code from here 
    }else if(finishedReading){ 
     //clean up if items left in array are less than 10 and also clear this interval function 
     console.log(new Date() + dataArray.length); 
     clearInterval(intervalFn); 
    } 
},interval) 

这里是区间函数的输出显示，数组后处理每10secs

Wed Dec 28 2016 14:46:09 GMT+0530 (India Standard Time)Machine Unit,Air Temperature �C,Water Temperature �C,Heat Temperature �C,Room Temperature �C,Date,Time,1,61,54,87,20,12/3/2016,8:39AM 
Wed Dec 28 2016 14:46:19 GMT+0530 (India Standard Time)2,41,57,92,21,29/9/2016,3:51PM,3,39,53,89,22,22/12/2016,5:30PM 
Wed Dec 28 2016 14:46:29 GMT+0530 (India Standard Time)4,56,27,87,23,9/7/2016,6:54AM,5,71,85,76,24,21/10/2016,7:29PM 
Wed Dec 28 2016 14:46:39 GMT+0530 (India Standard Time)6,82,64,85,25,22/3/2016,9:38PM,7,91,73,94,26,21/10/2016,1:30PM 
Wed Dec 28 2016 14:46:49 GMT+0530 (India Standard Time)8,35,79,89,27,30/4/2016,2:17PM,9,29,59,83,28,18/7/2016,7:49AM 

Answer 3

西科尔斯基的代码是相当不错的，并会工作，但看起来你是新的节点，所以我会将代码分解为组件，这不仅更容易理解，而且也是开发中的一个良好实践。

用于连接

function connectToMongo() { 
    mongoose.connect('mongo_url'); 
} 

用于断开

function disconnectToMongo() { 
    mongoose.connection.close() 
} 

为了节省数据

//Db is name of your mongo collection 
Db.create(data, function(err, data) { 
    if (err) { 
    console.log(err); return 
    } 
}); 

对于阅读整个CSV

function readDoc() { 
    var doc_content = []; 
    fs.createReadStream('data.csv').pipe(csv()) 
       .on('data',function(data){ 
        doc_content.push(data) 
       }) 
       .on('end',function(data){ 
        console.log('Read finished'); 
        return doc_content; 
       }) 
} 

现在，让我们在一个函数中使用这些功能，可以说which'll运行

function init() { 
    var numOfDocsToSave = 10; //just chnage here for num of docs you want to save in interval 
    var interval = 1000 * 10; //10 secs, thanks Sikorski for this 
    var data = readDoc(); 
    var chunk = []; 
    var intervalId = setInterval(function() { 
    connectToMongo(); 
    if(data.length >= numOfDocsToSave) { 
     chunk = data.splice(0, numOfDocsToSave); 
     saveInMongo(chunk); 
    } else { 
     saveInMongo(data); 
     clearInterval(intervalId); 
    } 
    disconnectoToMongo(); //we should not open connection for 5 minutes when we're not doing any operation 
    }, interval); 
    console.log("Interval ended"); 
} 

希望它可以帮助初始化。

更新

至于你说的db是未来不确定的，可能是因为你还没有创建它。

所以，你可以做到这一点，

〜model.js

var mongoose = require('mongoose'); 
mongoose.connect('mongodb://localhost/testDb'); 
module.exports = mongoose.model('Db', { 
    name: String //change argument with your schema 
}); 

现在在你的主文件导入此

var Db = require('./model'); //assuming your model.js and main file are in same directory