CODEWITHSUNDEEP
javascriptnode.js
Jason Chen
Jason Chen

Reputation: 2577

Nodejs unable to TCP/Parse long JSON file

I have a JSON file that is as follows. While the file itself seems unnecessarily long for an example, there is a reason I am including it. Basically, it appears to me that my file is too large to go through TCP to my live site. 

{
    "FormatNumber": 1,
    "Team_Types": [{
            "Teams": "EmmersonCod",
            "Channels": [{
                    "Team_Name": "Tanaka",
                    "Team_Members": 1,
                    "Team_Mascot": "Dolphin"
                },
                {
                    "Team_Name": "Drago",
                    "Team_Members": 2,
                    "Team_Mascot": "Lundgren"
                },
                {
                    "Team_Name": "Apollo",
                    "Team_Members": 3,
                    "Team_Mascot": "Crews"
                },
                {
                    "Team_Name": "Cobra",
                    "Team_Members": 4,
                    "Team_Mascot": "Kai"
                }
            ]
        },
        {
            "Teams": "Candy",
            "Channels": [{
                    "Team_Name": "Simson",
                    "Team_Members": 1,
                    "Team_Mascot": "The"
                },
                {
                    "Team_Name": "Rick",
                    "Team_Members": 2,
                    "Team_Mascot": "Sanchez"
                }
            ]
        },
        {
            "Teams": "FoxNews",
            "Channels": [{
                    "Team_Name": "David",
                    "Team_Members": 1,
                    "Team_Mascot": "Bannon"
                },
                {
                    "Team_Name": "Rickety",
                    "Team_Members": 2,
                    "Team_Mascot": "Crickett"
                },
                {
                    "Team_Name": "Lady",
                    "Team_Members": 3,
                    "Team_Mascot": "Madam"
                },
                {
                    "Team_Name": "Random",
                    "Team_Members": 4,
                    "Team_Mascot": "Words"
                },
                {
                    "Team_Name": "Put",
                    "Team_Members": 5,
                    "Team_Mascot": "Together"
                },
                {
                    "Team_Name": "To",
                    "Team_Members": 6,
                    "Team_Mascot": "Fill"
                },
                {
                    "Team_Name": "These",
                    "Team_Members": 7,
                    "Team_Mascot": "Blanks"
                },
                {
                    "Team_Name": "And",
                    "Team_Members": 8,
                    "Team_Mascot": "Illustrate"
                },
                {
                    "Team_Name": "The",
                    "Team_Members": 9,
                    "Team_Mascot": "Issues"
                },
                {
                    "Team_Name": "We",
                    "Team_Members": 10,
                    "Team_Mascot": "Are"
                },
                {
                    "Team_Name": "Going",
                    "Team_Members": 11,
                    "Team_Mascot": "Through"
                },
                {
                    "Team_Name": "At",
                    "Team_Members": 12,
                    "Team_Mascot": "This"
                },
                {
                    "Team_Name": "Very",
                    "Team_Members": 13,
                    "Team_Mascot": "Moment"
                },
                {
                    "Team_Name": "The",
                    "Team_Members": 14,
                    "Team_Mascot": "JSON"
                },
                {
                    "Team_Name": "Is",
                    "Team_Members": 15,
                    "Team_Mascot": "Too"
                },
                {
                    "Team_Name": "Long",
                    "Team_Members": 16,
                    "Team_Mascot": "For"
                },
                {
                    "Channl_Name": "My",
                    "Team_Members": 17,
                    "Team_Mascot": "TCP"
                },
                {
                    "Team_Name": "To",
                    "Team_Members": 18,
                    "Team_Mascot": "Go"
                }
            ]
        }
    ]
}

The Node.js code I am using to parse this is the following:

var express = require('express');
var net = require("net");
var fs = require("fs");
var request = require('request');
var app = express();
var server = net.createServer();
var pack;

app.use(function(req, res, next){
    res.header("Access-Control-Allow-Origin", "*");
    next();
});

server.on("connection", function(socket){
    pack = "";
    socket.setEncoding('utf8');
        socket.on("data", function(d){
            pack = JSON.parse(d);
            console.log(pack.Timestamp.LocalTimestamp);
            app.set('dee', d);
                app.get("/"+pack.FormatNumber, function(req, res){
                    res.writeHead(200, {'Content-Type': 'text/plain'});
                    res.write(req.app.get('dee'));
                    res.end();
                });
        });
        socket.once("close", function(){
            console.log("connection closed");
        });
});

server.on("error", function(){
    console.log("connection error");
});

server.listen(9000, function(){
    console.log("Server Listening to Port 9000");
});

app.listen(8081, function(){

});

This JSON file gets sent as a TCP packet to my Node server, which in turn parses it. The code and functionality works perfectly on my localhost environment. However, when attempting to push my JSON file over my Digitalocean Node.js server, I get the following error.

undefined:57
                                        "Cha
                                         ^

SyntaxError: Unexpected token C
    at Object.parse (native)
    at Socket.<anonymous> 
    at emitOne (events.js:77:13)
    at Socket.emit (events.js:169:7)
    at readableAddChunk (_stream_readable.js:146:16)
    at Socket.Readable.push (_stream_readable.js:110:10)
    at TCP.onread (net.js:523:20)

If I reduce the amount of objects in my JSON array, then passing it to my live server works as intended. I am unclear as to why this is. The TCP packet only parses through a smaller JSON file.

Upvotes: 4

Views: 434

Answers (1)

josh3736
josh3736

Reputation: 145162

TCP provides a stream of bytes, not messages.

Despite the fact that TCP sends its data over IP packets, TCP is not a packet protocol. A TCP socket is simply a stream of data. Thus, it is incorrect to view the data event as a logical message. In other words, one socket.write on one end does not equate to a single data event on the other. A single data event might contain multiple messages, a single message, or only part of a message.

The good news is this is a problem already solved many times over. I'd recommend either:

  • Using a library meant for passing JSON messages over TCP.
  • Using something like redis as a pub-sub messaging solution (this option makes your app much easier to scale)
  • If you know that your two apps will always run on the same machine, you should use node's built-in IPC mechanism.

Upvotes: 5

Related Questions