I'm trying to read a .txt file from 3.5GB line-by-line and save in the db mongo. Columns use the "|" tab. and I'm using Node js with Express. I did the test with a file smaller than 43 lines and everything worked fine but with the largest file I am not getting and I do not get an error message. I already managed to save the same file in MySql but with Mongo I can not. The request takes a little longer then the server crashes and nothing happens. The code is as follows:
const express = require('express');
const router = express.Router();
const Salario = require('../models/Salarios');
const lineReader = require('line-reader');
let i = 0;
router.get('/import', function(req, res) {
lineReader.eachLine('./teste.txt', function (line, last) {
let str = line;
let arr = str.split("|");
let salario = new Salario();
salario.cd_ugestora = arr[0];
salario.de_ugestora = arr[1];
salario.de_cargo = arr[2];
salario.de_tipocargo = arr[3];
salario.cd_cpf = arr[4];
salario.dt_mesanorefencia = arr[5];
salario.no_servidor = arr[6];
salario.vl_vantagens = arr[7];
salario.de_uorcamentaria = arr[8];
salario.save(function(err) {
if (err) res.send(err);
console.log(i + " - adicionado!");
i++;
});
if (last) {
res.json({ message: 'Importado com sucesso!' });
}
});
});
Has anyone ever been through this?