node.js - mongodb中数据find出来，提示process out of memory，数据大小并没有超过NodeJS默认的512MB

Question

使用mongoose从数据库中一次性find出数据，提示：

但是我查了default the memory limit of Node.js is 512 mb，我的数据集合大小只有127MB，并没有超过这个大小，也不需要设置--max_old_space_size吧，求解

代码如下：

//model.js
var LagouInfo = require('./schema.js')
var DesData = require('./des.js');
var async = require('async');

LagouInfo.find({}, function(res){
    res.forEach(function(item){
        if(item.content.trim()){
            var sumSalary = 0;
            item.salary.forEach(function(sitem){
                var num;
                if(sitem.indexOf('-') == -1){
                    num = sitem.replace(/\D+/, '');
                }else{
                    num = parseInt(sitem.trim().split('-')[1]);
                }
                // console.log(sitem, num);
                sumSalary += num;
            })
            var tags = item.tag.trim().split(',');
            tags.forEach(function(tag){
                DesData.update({'tag': tag}, {$push: {'content': item.content}});
                DesData.update({'tag': tag}, {$inc: {'total': item.total}});
                DesData.update({'tag': tag}, {$inc:{'salary': sumSalary}});
            })

        }
    })
})

//des.js
var mongoose = require('./db.js'),
    Schema = mongoose.Schema;

var InfoSchema = new Schema({          
    tag: {type: String},
    content: {type: Array},
    total: {type: Number},
    salary: {type: Number},     
});
var Data = mongoose.model('desdata', InfoSchema, 'desdata');
function insert(obj, callback){
    var data = new Data(obj);
    data.save(function(err, res){
        if(err) console.log('Error:' + err);
        else callback(null, res);
    })
}
function update(conditions, updateStr){
    Data.update(conditions, updateStr, function(err, res){
        if(err) console.log('Error:' + err);
        else console.log('Update Success');
        // else callback(null, res);
    })
}
function find(conditions, callback){
    Data.find(conditions, function(err, res){
        if(err) console.log('Error:' + err);
        else callback(res);
    })
    // return Data.find(conditions).exec();
}
module.exports = {
    insert: insert,
    update: update,
    find: find
}

//schema.js
var mongoose = require('./db.js'),
    Schema = mongoose.Schema;
var LagouSchema = new Schema({
    name: {type: String},
    cid: {type: Number},
    process: {type: String},
    content: {type: String},
    url: {type: String},
    tag: {type: String},
    total: {type: Number},
    salary: {type: Array}
});
var Lagou = mongoose.model('lagou', LagouSchema, 'lagou');

function update(conditions, update){
    Lagou.update(conditions, update, function(err, res){
        if(err) console.log('Error:' + err);
        else console.log('Res:' + res);
    })
}
function del(conditions){
    Lagou.remove(conditions, function(err, res){
        if(err) console.log('Error:' + err);
        else console.log('Res:' + res);
    })
}
function find(conditions, callback){
    Lagou.find(conditions, function(err, res){
        if(err) console.log('Error:' + err);
        else callback(res);
    })
}
module.exports = {
    find: find,
    del: del,
    update: update
}

//db.js
var mongoose = require('mongoose'),
    DB_URL = 'mongodb://localhost:27017/result';

mongoose.Promise = global.Promise;
mongoose.connect(DB_URL);
//连接成功
mongoose.connection.on('connected', function(){
    console.log('Mongoose connection open to ' + DB_URL);
})
//连接异常
mongoose.connection.on('error', function(err){
    console.log('Mongoose connection error: ' + err);
})
//连接断开
mongoose.connection.on('disconnected', function(){
    console.log('Mongoose connection disconnected');
})

module.exports = mongoose;

============更新2017-01-12===================

尝试设置了--max_old_space_size还是报错

报错内容：

$ node --max_old_space_size=2000 models/test.js
Mongoose connection open to mongodb://localhost:27017/result

<--- Last few GCs --->

  251717 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1564.0 / 0 ms [allocation failure] [scavenge might not succeed].
  253271 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1554.2 / 0 ms [allocation failure] [scavenge might not succeed].
  254868 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1597.0 / 0 ms [last resort gc].
  256434 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1566.2 / 0 ms [last resort gc].


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 00000394003B4639 <JS Object>
    2: /* anonymous */(aka /* anonymous */) [D:\software\self_learn\Paper\preprocess\node_modules\mongodb-core\lib\connection\pool.js:~1096] [
pc=000002BA3DA6C022] (this=00000394003041B9 <undefined>)
    3: waitForAuth(aka waitForAuth) [D:\software\self_learn\Paper\preprocess\node_modules\mongodb-core\lib\connection\pool.js:1088] [pc=000002
BA41DE1FBC] (this=00000394003041B9 <undefined>,cb=0000...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

Answer 1

I feel that this error has nothing to do with mongodb. It is probably a memory problem in the node program

Can you try this?

node --max_old_space_size=2000 server.js

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

Answer 2

First of all, the data size seen in the database is different from the volume read by the program into the memory. It is recommended to add some conditions to reduce the amount of data and see if there are any problems? Then add more data to see if there are any problems? See how many records there are when there is a problem.
Second, why do I think there is something wrong with the writing of your function? DesData.update is an asynchronous method. You are using a synchronous writing method. Then I see that your tags.foreach method should be looped multiple times. I don’t know how many times, but it should be quite large, which may cause your oom problem. Here----there are too many asynchronous methods waiting to be executed. And is it really okay for you to write asynchronously like this? It is recommended to use the async package to change your code. It is best to use forEachLimit. I won’t directly change the code for you while typing on the phone. If it still doesn’t work, I will change it for you when I am on the computer.

Answer 3

Can you post part of your code to see if there are any potential issues in the code?