Using "fs" to read JSON file throws "invalid json" Node.js

Question

This makes me crazy. I don't know why but when I use the filesystem require('fs') to read valid Json file I get error: Invalid json. Here it is the code:

var fs = require('fs');
const FILE_NAME = './json.json'

/*
// This way it works!
var jsonData = require('./json.json')
console.log(jsonData);
*/

async function readFile(filePath){
    return new Promise(function(resolve, reject){
        fs.readFile(filePath, 'utf8', function(err, contents) {
            if(err){
              console.log("Cant read the file " + err);
              reject(err)
            }
            else{
              resolve(contents)
            }
        })
    })
}

async function getNames(fileName){
  readFile(fileName).then(function(data){
    try {
      console.log(`Type of data: ` + typeof data);
      console.log("File data: " + data);
      return JSON.parse(data);
    } catch (error) {
      throw new Error( "Invalid JSON: " + error);
    }
  }).then(function(data){
    console.log(`FILE OBJECT: ` + data);
  }).catch(function(err){
    console.error(err);
  })
}
getNames(FILE_NAME)

This is the file content:

{
  "name": "riko"
}

This is the console output:

Type of data: string
File data: {
  "name": "riko"
}
Error: Invalid JSON: SyntaxError: Unexpected token  in JSON at position 0
    at C:\Users\rojer\Desktop\Node\test\main.js:31:13

I know I could use var jsonData = require('./json.json'), but

1. I want to know the reason it doesn't work.
2. What if the JSON data is embedded somewhere in a regular text file.

There seems to be some garbage.

Please help.

Answer 1

This:

Error: Invalid JSON: SyntaxError: Unexpected token  in JSON at position 0
    at C:\Users\rojer\Desktop\Node\test\main.js:31:13

tells us that there's an invisible character at the beginning of the file, probably a byte order mark (BOM), that require is handling but your code isn't. If the file is really in UTF-8, that BOM will be \xEF\xBB\xBF. Once read as UTF-8 into a JavaScript string, that will be the code point \u{FEFF} (because JavaScript strings are UTF-16 [but tolerate invalid surrogate pairs]). Update: Your binary listing of it confirms that.

I can confirm that if I have a UTF-8 JSON file with a BOM, require reads it and handles the BOM, but readFile returns the contents of the with the BOM intact, which trips up JSON.parse.

You can check for the BOM and strip it off, see *** lines:

const UTF8_BOM = "\u{FEFF}";                      // ***
async function getNames(fileName){
  readFile(fileName).then(function(data){
    try {
      console.log(`Type of data: ` + typeof data);
      if (data.startsWith(UTF8_BOM)) {            // ***
          data = data.substring(UTF8_BOM.length); // ***
      }
      console.log("File data: " + data);
      return JSON.parse(data);
    } catch (error) {
      throw new Error( "Invalid JSON: " + error);
    }
  }).then(function(data){
    console.log(`FILE OBJECT: ` + data);
  }).catch(function(err){
    console.error(err);
  })
}

---

Alternately, if you don't want the BOM there, here's a quick and dirty tool to add/remove BOMs on UTF-8 files:

const fs = require("fs");

const UTF8_BOM = "\u{FEFF}";

const actions = new Map([
    ["-a", addBOM],
    ["-r", removeBOM],
    ["-t", toggleBOM]
]);

main();

function main() {
    const filename = process.argv[2];
    const option = process.argv[3] || "-t";
    const action = actions.get(option);

    if (!filename) {
        help();
        return;
    }

    if (!action) {
        console.error(`Invalid option ${option}`);
        help();
        return;
    }

    fs.readFile(filename, 'utf-8', (err, data) => {
        if (err) {
            console.error(`${filename}: Error reading file: ${err}`);
            return;
        }
        const hasBOM = data.startsWith(UTF8_BOM);
        action(filename, data, hasBOM);
    });
}

function writeResult(filename, data, toggle, successMessage, failMessage) {
    fs.writeFile(filename, data, "utf-8", (err) => {
        if (err) {
            console.error(`${filename}: ${failMessage}: ${err}`);
        } else {
            console.log(`${filename}: ${successMessage}${toggle ? " (toggled)" : ""}`);
        }
    });
}

function addBOM(filename, data, hasBOM, toggle) {
    if (hasBOM) {
        console.log(`${filename}: Already has a BOM`);
    } else {
        writeResult(filename, UTF8_BOM + data, toggle, "Added BOM", "Error adding BOM");
    }
}

function removeBOM(filename, data, hasBOM, toggle) {
    if (!hasBOM) {
        console.log(`${filename}: Already doesn't have a BOM`);
    } else {
        writeResult(filename, data.substring(UTF8_BOM.length), toggle, "Removed BOM", "Error removing BOM");
    }
}

function toggleBOM(filename, data, hasBOM) {
    if (hasBOM) {
        removeBOM(filename, data, hasBOM, true);
    } else {
        addBOM(filename, data, hasBOM, true);
    }
}

function help() {
    console.log("Usage: node utf8bomtoggle [filename] {options}");
    console.log("{options} can be:");
    console.log("  -t   Toggle a BOM [default]");
    console.log("  -a   Add a BOM if not present");
    console.log("  -r   Remove a BOM if present");
}