CODEWITHSUNDEEP
jsonnode.jsimportcouchdb
MikeB
MikeB

Reputation: 788

couchdb - Import json file

I have a file with data in the json format. The data is odd in that the rows have varying subsets of a set of properties that are not all known in advance (over time they will build up). For example:

[{"firstName":"Joe","education":"highschool","sex":"male"},
 {"lastName":"Edwards","address":"ohio","sex":"male"},
{"favoriteSport":"cycling","bicycle":"raleigh","ownsBoat":"yes","ownsDog":"yes","ownsHouse":"yes"}]

A large amount of data exists already in a file so I would like to import it in to couchdb rather than enter the data item by item. I followed the procedures from a post here but, while a db was created, it was empty. I used:

curl -X PUT -d @../Data/data.json http://127.0.0.1:5984/test_import

UPDATE: Since I'm working with nodejs (newbie), I thought I'd try using 'cradle'. My thought was to take the import the data as an array, and bulk load that using 'cradle's dbsave(). But using the following:

var fs = require('fs');
var cradle = require('cradle');

var data = fs.readFile( '../Data/data.json', function (err, data) {
  if (err) {
    throw err; 
  }
.
.
.
makeDB(bigdata,'test_import');  // where 'bigdata' is an array of json objects/couchdb 'documents'
});

function makeDB (p,filename) {

var db = new(cradle.Connection)().database(filename);
console.log(db);

db.save(p, function(err,res) {
if (err) {
    console.log(err);
} else {
    console.log('Success!');
}
});

};

The latter seems to work!! A database is created and filled but it does however throw the following errors:

k:\nodejs\node_modules\cradle\lib\cradle.js:198
            return callback(body);
                   ^
TypeError: undefined is not a function

OR

k:\nodejs\node_modules\cradle\lib\cradle.js:201
        callback(null, self.options.raw ? body : new cradle.Response(body, res
        ^
TypeError: undefined is not a function
    at Request.cradle.Connection.request [as _callback] (k:\nodejs\node_modules\cradle\lib\cradle.
js:201:9)
    at Request.init.self.callback (k:\nodejs\node_modules\request\main.js:120:22)
    at Request.EventEmitter.emit (events.js:91:17)
    at Request.<anonymous> (k:\nodejs\node_modules\request\main.js:633:16)
    at Request.EventEmitter.emit (events.js:88:17)
    at IncomingMessage.Request.start.self.req.self.httpModule.request.buffer (k:\nodejs\node_modul
es\request\main.js:595:14)
    at IncomingMessage.EventEmitter.emit (events.js:115:20)
    at IncomingMessage._emitEnd (http.js:366:10)
    at HTTPParser.parserOnMessageComplete [as onMessageComplete] (http.js:149:23)
    at Socket.socketOnData [as ondata] (http.js:1366:20)

Upvotes: 1

Views: 2605

Answers (1)

MikeB
MikeB

Reputation: 788

[SOLVED]

The answers to my questions are that yes, of course, couchdb is perfectly suited to that kind of data. The easiest way I found to do the bulk import with node.js is using cradle (whose creator provided solution to problem). The preceding code works error free with the following changes to the makeDB function:

//  Take the array of objects and create a couchdb database

function makeDB (data,filename) {

var db = new(cradle.Connection)().database(filename);
//console.log(db);
db.create(function(err){
    if (err) console.log(err); 
    });
db.save(data, function(err) {
    if (err) console.log(err);
    console.log(filename + ' is created.');
    });

};

Upvotes: 1

Related Questions