Best way to reduce object data sent from browser to server

Question

We have the following data sent from browser to server, what is the best way to serialize/deserialize the data other than JSON?

We save the data initially in the client browser and we sent it to the server at regular checkpoints. Due to the size of the data, it is taking high browser memory as well as a network when sending the data. We want to reduce the size of the data sent to the server as the keys are mostly would be the same for each object but values changes.

[
   {
      "range":{
         "sLineNumber":3,
         "sColumn":3,
         "eLineNumber":3,
         "eColumn":3
      },
      "rLength":0,
      "text":"\n",
      "rOffset":4,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":4,
         "sColumn":1,
         "eLineNumber":4,
         "eColumn":1
      },
      "rLength":0,
      "text":"\n",
      "rOffset":5,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":5,
         "sColumn":1,
         "eLineNumber":5,
         "eColumn":1
      },
      "rLength":0,
      "text":"\n",
      "rOffset":6,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":1,
         "eLineNumber":6,
         "eColumn":1
      },
      "rLength":0,
      "text":"f",
      "rOffset":7,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":2,
         "eLineNumber":6,
         "eColumn":2
      },
      "rLength":0,
      "text":"a",
      "rOffset":8,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":3,
         "eLineNumber":6,
         "eColumn":3
      },
      "rLength":0,
      "text":"s",
      "rOffset":9,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":4,
         "eLineNumber":6,
         "eColumn":4
      },
      "rLength":0,
      "text":"d",
      "rOffset":10,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":5,
         "eLineNumber":6,
         "eColumn":5
      },
      "rLength":0,
      "text":"f",
      "rOffset":11,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":6,
         "eLineNumber":6,
         "eColumn":6
      },
      "rLength":0,
      "text":"a",
      "rOffset":12,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":7,
         "eLineNumber":6,
         "eColumn":7
      },
      "rLength":0,
      "text":"s",
      "rOffset":13,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":8,
         "eLineNumber":6,
         "eColumn":8
      },
      "rLength":0,
      "text":"f",
      "rOffset":14,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":9,
         "eLineNumber":6,
         "eColumn":9
      },
      "rLength":0,
      "text":"s",
      "rOffset":15,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":10,
         "eLineNumber":6,
         "eColumn":10
      },
      "rLength":0,
      "text":"a",
      "rOffset":16,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":11,
         "eLineNumber":6,
         "eColumn":11
      },
      "rLength":0,
      "text":"f",
      "rOffset":17,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":12,
         "eLineNumber":6,
         "eColumn":12
      },
      "rLength":0,
      "text":"s",
      "rOffset":18,
      "rMoveMarkers":false
   }
]

One of the ways we thinking is sending them as an array without a key as we know the position of each key. Not sure if there is any package available for this conversion.

Answer 1

Here are few ways how to make the best compression of HTTP response in your app:

• Use GZIP compression - that compress any data that you send from server to client. For here is a library for express.js. Should be used together with other utilities.
• Use Google Protobuf. As authors said - it is language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler.
• Use MessagePack - is an efficient binary serialization format for JSON. "It's like JSON, but fast and small". Simple to integrate on any platform.

Also if you can update data by itself you can:

• if keys are always the same - remove keys and leave only values
• if keys can be different - 2 options:
  • structure with 2 attributes: list of keys and list of list with values

    {
       "keys": ["sLineNumber", "sColumn", "eLineNumber", ..., "rMoveMarkers"],
       "values": [
           [3, 4, ...],
           [3, 1, ...],
           [3, 4, ...],
           ...
           [false, false, ...],
       ]
    }
    

  • dict with key and value with all values:

    {
        "sLineNumber": [3, 4, ...],
        "sColumn":     [3, 1, ...],
        "eLineNumber": [3, 4, ...],
        ...
        "rMoveMarkers":[false, false, ...]
    }
    

  • or even just have list of list as a response.

Together with gZip - it should be enough for most cases.

Answer 2

We want to reduce the size of the data sent to the server as the keys are mostly would be the same for each object but values changes.

If you consider your data as a table, you could represent each column as an object property, with row values in arrays, like this:

{
  "sLineNumber": [3, 4,         /* ... */ ],
  "sColumn":     [3, 1,         /* ... */ ],
  "eLineNumber": [3, 4,         /* ... */ ],
  "eColumn":     [3, 1,         /* ... */ ],
  "rLength":     [0, 0,         /* ... */ ],
  "text":        ["\n", "\n",   /* ... */ ],
  "rOffset":     [4, 5,         /* ... */ ],
  "rMoveMarkers":[false, false, /* ... */ ]
}

This structure keeps all property names (except "range") and is far more memory efficient.

The data could still be serialized as JSON, at only about 25% the size of your original structure.

You could get an additional size reduction by using integers (1 and 0) instead of booleans (true and false) in "rMoveMarkers".

Quick and dirty code for restructuring your data in the snippet below:

var data = [
   {
      "range":{
         "sLineNumber":3,
         "sColumn":3,
         "eLineNumber":3,
         "eColumn":3
      },
      "rLength":0,
      "text":"\n",
      "rOffset":4,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":4,
         "sColumn":1,
         "eLineNumber":4,
         "eColumn":1
      },
      "rLength":0,
      "text":"\n",
      "rOffset":5,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":5,
         "sColumn":1,
         "eLineNumber":5,
         "eColumn":1
      },
      "rLength":0,
      "text":"\n",
      "rOffset":6,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":1,
         "eLineNumber":6,
         "eColumn":1
      },
      "rLength":0,
      "text":"f",
      "rOffset":7,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":2,
         "eLineNumber":6,
         "eColumn":2
      },
      "rLength":0,
      "text":"a",
      "rOffset":8,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":3,
         "eLineNumber":6,
         "eColumn":3
      },
      "rLength":0,
      "text":"s",
      "rOffset":9,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":4,
         "eLineNumber":6,
         "eColumn":4
      },
      "rLength":0,
      "text":"d",
      "rOffset":10,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":5,
         "eLineNumber":6,
         "eColumn":5
      },
      "rLength":0,
      "text":"f",
      "rOffset":11,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":6,
         "eLineNumber":6,
         "eColumn":6
      },
      "rLength":0,
      "text":"a",
      "rOffset":12,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":7,
         "eLineNumber":6,
         "eColumn":7
      },
      "rLength":0,
      "text":"s",
      "rOffset":13,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":8,
         "eLineNumber":6,
         "eColumn":8
      },
      "rLength":0,
      "text":"f",
      "rOffset":14,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":9,
         "eLineNumber":6,
         "eColumn":9
      },
      "rLength":0,
      "text":"s",
      "rOffset":15,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":10,
         "eLineNumber":6,
         "eColumn":10
      },
      "rLength":0,
      "text":"a",
      "rOffset":16,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":11,
         "eLineNumber":6,
         "eColumn":11
      },
      "rLength":0,
      "text":"f",
      "rOffset":17,
      "rMoveMarkers":false
   },
   {
      "range":{
         "sLineNumber":6,
         "sColumn":12,
         "eLineNumber":6,
         "eColumn":12
      },
      "rLength":0,
      "text":"s",
      "rOffset":18,
      "rMoveMarkers":false
   }
];

function transform(data){
  var transformed = {};
  ['rLength', 'text', 'rOffset', 'rMoveMarkers']
    .map(x => transformed[x] = data.map(y => y[x]));
  ['sLineNumber', 'sColumn', 'eLineNumber', 'eColumn']
    .map(x => transformed[x] = data.map(y => y.range[x]));
  return transformed;
}

var originalLength = JSON.stringify(data).length;
var transformedLength = JSON.stringify(transform(data)).length;

console.log(
  'Reduced to ' + 
  (100 * transformedLength / originalLength).toFixed(1) + 
  '% size of original, from ' + originalLength + ' characters to ' + 
  transformedLength + ' characters.'
);
console.log(transform(data));

One of the ways we thinking is sending them as an array without a key as we know the position of each key. Not sure if there is any package available for this conversion.

Reducing your data to a two-dimensional array can be done like this:

function transform(data) {
  return ['rLength', 'text', 'rOffset', 'rMoveMarkers']
    .map(x => data.map(y => y[x]))
    .concat(
      ['sLineNumber', 'sColumn', 'eLineNumber', 'eColumn']
      .map(x => data.map(y => y.range[x]))
    );
}

Note that dropping the keys (property names) only gives a marginal reduction compared to the data structure above, while it makes it harder to track bugs or troubleshoot any issues, as you have to make sure you don't accidentally mix up which array belongs to which key.

Demo code in the snippet below:

var data = [{
    "range": {
      "sLineNumber": 3,
      "sColumn": 3,
      "eLineNumber": 3,
      "eColumn": 3
    },
    "rLength": 0,
    "text": "\n",
    "rOffset": 4,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 4,
      "sColumn": 1,
      "eLineNumber": 4,
      "eColumn": 1
    },
    "rLength": 0,
    "text": "\n",
    "rOffset": 5,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 5,
      "sColumn": 1,
      "eLineNumber": 5,
      "eColumn": 1
    },
    "rLength": 0,
    "text": "\n",
    "rOffset": 6,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 1,
      "eLineNumber": 6,
      "eColumn": 1
    },
    "rLength": 0,
    "text": "f",
    "rOffset": 7,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 2,
      "eLineNumber": 6,
      "eColumn": 2
    },
    "rLength": 0,
    "text": "a",
    "rOffset": 8,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 3,
      "eLineNumber": 6,
      "eColumn": 3
    },
    "rLength": 0,
    "text": "s",
    "rOffset": 9,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 4,
      "eLineNumber": 6,
      "eColumn": 4
    },
    "rLength": 0,
    "text": "d",
    "rOffset": 10,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 5,
      "eLineNumber": 6,
      "eColumn": 5
    },
    "rLength": 0,
    "text": "f",
    "rOffset": 11,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 6,
      "eLineNumber": 6,
      "eColumn": 6
    },
    "rLength": 0,
    "text": "a",
    "rOffset": 12,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 7,
      "eLineNumber": 6,
      "eColumn": 7
    },
    "rLength": 0,
    "text": "s",
    "rOffset": 13,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 8,
      "eLineNumber": 6,
      "eColumn": 8
    },
    "rLength": 0,
    "text": "f",
    "rOffset": 14,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 9,
      "eLineNumber": 6,
      "eColumn": 9
    },
    "rLength": 0,
    "text": "s",
    "rOffset": 15,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 10,
      "eLineNumber": 6,
      "eColumn": 10
    },
    "rLength": 0,
    "text": "a",
    "rOffset": 16,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 11,
      "eLineNumber": 6,
      "eColumn": 11
    },
    "rLength": 0,
    "text": "f",
    "rOffset": 17,
    "rMoveMarkers": false
  },
  {
    "range": {
      "sLineNumber": 6,
      "sColumn": 12,
      "eLineNumber": 6,
      "eColumn": 12
    },
    "rLength": 0,
    "text": "s",
    "rOffset": 18,
    "rMoveMarkers": false
  }
];

function transform(data) {
  return ['rLength', 'text', 'rOffset', 'rMoveMarkers']
    .map(x => data.map(y => y[x]))
    .concat(
      ['sLineNumber', 'sColumn', 'eLineNumber', 'eColumn']
      .map(x => data.map(y => y.range[x]))
    );
}

var originalLength = JSON.stringify(data).length;
var transformedLength = JSON.stringify(transform(data)).length;

console.log(
  'Reduced to ' +
  (100 * transformedLength / originalLength).toFixed(1) +
  '% size of original, from ' + originalLength + ' characters to ' +
  transformedLength + ' characters.'
);
console.log(transform(data));

Answer 3

I would suggest creating a WebSocket connection to the server and then pushing the data over that as soon as it has been created. If you JS is streamlined enough then you should easily be able to send 1,000 these objects per second.

Checkout https://socket.io/ which is the easy way to use WebSockets.

The other limiting factor is going to be your browser code. You will likely need to optimise that as well, if your using React make sure to work on stopping it making needs rerenders.

Answer 4

Activate gzip compression and send it as array without keys.