"nedb" insert csv bulk
require('dotenv').load()
const fs = require('fs')
const csv = require('csv-parse')
const es = require('event-stream')
const Datastore = require('nedb')
const BatchStream = require('batch-stream')
const db = new Datastore({ filename: process.env.DB_PATH, autoload: true })
const debug = require('debug')('setup')
function massage ([ipLo, ipHi, cc, country, area, city, lat, lng]) {
return { ipLo, ipHi, cc, country, area, city, lat, lng }
}
function setup () {
let qty = 0
return new Promise((resolve, reject) => {
fs.createReadStream(process.env.IP2LOCATION_PATH)
// read and parse csv
.pipe(csv())
// batch it up
.pipe(new BatchStream({ size: 100 }))
// write it into the database
.pipe(es.map((batch, cb) => {
// massage and persist it
db.insert(batch.map(massage), _ => {
qty += batch.length
if (qty % 100 === 0)
debug(`Inserted ${qty} documents…`)
cb.apply(this, arguments)
})
}))
.on('end', resolve)
.on('error', reject)
})
}
module.exports = setup
if (!module.parent) {
debug('Setting up geo database…')
setup()
.then(_ => debug('done!'))
.catch(err => debug('there was an error :/', err))
}