How to write a JSON file based on data from multiple markdown files

Question

I need to build an API with the data from my markdown files.

I have approximately 100 markdown files with data like this in the top of file:

---
title: Lorem ipsum
category: component
primaryKeywords: curved horizon
secondaryKeywords: css block content seperator
---

Desired output is a single .json file with all data from my .md file as objects in an array.

Example:

[
  {
    "title": "Lorem ipsum",
    "category": "component",
    "primaryKeywords": "curved horizon",
    "secondaryKeywords": "css block content seperator"
  },
  {
    "title": "Lorem ipsum",
    "category": "component",
    "primaryKeywords": "curved horizon",
    "secondaryKeywords": "css block content seperator"
  },
  {
    "title": "Lorem ipsum",
    "category": "component",
    "primaryKeywords": "curved horizon",
    "secondaryKeywords": "css block content seperator"
  }
]

The JSON file should be generated as part of my frontend build. I am using gulp.

I have tried doing something like this:

gulp.task('search-api', function (cb) {
  const fs = require('fs');
  const matter = require('gray-matter');
  const str = fs.readFileSync('./src/docs/01-Components/02-Layout/03-bow.md', 'utf8');
  console.log(matter(str));
});

I am able to show data from 1 file in the console doing this. But i need help to show data from all files in ./src/docs/ and then merge it to 1 result and parse it to 1 JSON file.

How can I do this? All help and suggestions are appreciated.

Answer 1

I did this as an "exercise" using node and not gulp:

const fs = require('fs');
const glob = require('glob');
const os = require('os');

const markdownFiles = glob.sync("./markdown/*.md");  // an array of files in the 'markdown' directory
let finalArray = [];


function buildJSONfile() {

  let contents;

  markdownFiles.forEach((nextFile) => {

    contents = fs.readFileSync(nextFile, "UTF8");

        // os.EOL = \r\n for windows, \n for POSIX
    let grayMatter = contents.match(new RegExp(`---${os.EOL}((.*${os.EOL})*?)---`));  // get just the content between the "---"s in array[1]

    //  add quotes around all the keys and values and add commas between key:value pairs
    let addedQuotes = grayMatter[1].replace(/^([^:]*)(:\s*)(.*)(\s*)/mg, '"$1"$2"$3",');  

    addedQuotes = addedQuotes.slice(0, addedQuotes.length - 1);  // remove the extra comma at the end of the last value

    let addedBrackets = `{${addedQuotes}}`;  // add brackets around the whole thing so that we can use JSON.parse

    let JSONobject = JSON.parse(addedBrackets);

    finalArray.push(JSONobject);
  });

      // write to a file : result.json
  fs.writeFileSync("./result.json", JSON.stringify(finalArray, null, '\t'), "UTF8");
};

buildJSONfile();  // comment out if using gulp

Run with node yourFileNameHere.js.
You could also put it into a gulpfile.js and run it via gulp buildJSONfile.

Answer 2

Basically what you want to do is

1. create a writeStream
2. add "["

3. Add the Object

4. loop for every file: Add the comma ","

5. Add another Object
   :loopend
6. Add another closing "]"

For this you can use the walk package.

const fs = require('fs');
const walk = require('walk');
const matter = require('gray-matter');
const dirname = "./src/docs";
const path = require('path');
const walker = walk.walk(dirname);
let prefix = ""
const stream = fs.createWriteStream("json.json", {flags:'a'});
stream.write("[\n");
walker.on("file",  (root, fileStats, next) => {
    const str = fs.readFileSync(path.join(root, fileStats.name), 'utf8');
    stream.write(prefix);
    stream.write(JSON.stringify(matter(str),null, 4));
    prefix=","
    next();
});

walker.on("errors", function (root, nodeStatsArray, next) {
    next();
});

walker.on("end", function () {
    stream.write("\n]");
    stream.end();
});

PS: This code I have done from top of my head to give you a hint of some sort. Feel free to edit if there are bugs.