Gulp merge json files from different folders while keeping folder structure

Question

I would like to merge multiple .json files from different folders while keeping the general folder structure.

Input

/lang/module1/file1.json
/lang/module1/file2.json
/lang/module2/file1.json
/lang/module2/subModule1/file1.json

/lang_additional/module1/file1.json
/lang_additional/module1/file2.json
/lang_additional/module2/file1.json
/lang_additional/module2/subModule1/file1.json

Expected Output

/dist/lang/module1/file1.json                (both file1.json's merged)
/dist/lang/module1/file2.json                (both file2.json's merged)
/dist/lang/module2/file1.json                (both file1.json's merged)
/dist/lang/module2/subModule1/file1.json     (both file1.json's merged)

Is it possible to achieve this with gulp? There may be multiple folders and subfolders. I wish to designate which parent folder has precedence in the merge conflict resolution.

Answer 1

Might be useful, I solved the same problem:

folder-1-localization:
 - en.json 
 - pl.json
 - es.json

folder-2-localization:
 - en.json 
 - pl.json
 - es.json  

and expect to get merged (folder 1 + folder 2):
 - en.json 
 - pl.json
 - es.json

The idea is that we create a stream and read files from the first folder. Then we iterate through first folder's files using flatmap and merge the stream of each file with another stream from one file in second folder (it is important here that the names match) using merge(). Lastly we combine files from different folders into one by jsonMerge.

const gulpFlatmap = require('gulp-flatmap')
const jsonMerge = require('gulp-merge-json')
const merge = require('merge-stream')

const FOLDER_1_LOC = './folder-1-localization/**/*.json'
const FOLDER_2_LOC_PATH = './folder-2-localization'
const RESULT_LOC = './output/localization'

task('merge-localization', () => {
  return src(FOLDER_1_LOC).pipe(gulpFlatmap((stream, file) => {
    const localizationFilename = file.basename
    return merge(src(`${FOLDER_2_LOC_PATH}/${localizationFilename}`), stream).pipe(jsonMerge({ fileName: localizationFilename, jsonSpace: '  ' }))
  })).pipe(dest(RESULT_LOC)
})

Answer 2

This works for your example input and I believe any number of subfolders in each folder. If there is a matching file in another directory it is added to a streams array that will then be used to start the necessary number of gulp streams to send to 'gulp-merge-json' and your 'dist' output with the folder structure retained.

Use with "gulp mergeJSON"

var gulp = require('gulp');
var fs = require('fs');
var path = require('path');
var merge = require('gulp-merge-json');

const files = [];
const parentFolders = [];
let   streams = [];
const baseNames = [];

//    note that the way 'gulp-merge-json' works is the last file having the same key as an earlier file 'wins' on the merge
//    so whichever directory is listed last in the folders array will have PRECEDENCE
//   'gulp-merge-json' will also take an 'edit' function option

// [  in the comments you said you wanted the 'lang' folder structure preserved in './dist'
//     but with 'lang_additional' precedence  ]

//  By the way, this works with more than two directories or it can work with
// folders of different levels such as:
//         const folders = ['lang', 'lang_additional/module1/subModule2'];
// only merging those files with the same directory structure starting at the deepest
// levels only. So in the above example, only within subModule2 and below.

const folders = ['lang', 'lang_additional'];

gulp.task('mergeJSON', function () {

  getFiles(folders);
  makeStreams();

    // streams[1] =   lang\module1\file2.json,   lang_additional\module1\file2.json

  // spin up multiple "streams", not really a stream yet until gulp.src creates one
  streams.forEach(function (stream) {

    //  get the fileName from one of the stream files, they all end with the same file
    let fileName = path.basename(stream[stream.length - 1]);

    //  get the directories of one of the streams, again all files within a stream have the same directories
    let dirName = path.dirname(stream[stream.length - 1]);
    // strip off the first directory, leaving all subdirectories
    dirName = dirName.substr(dirName.indexOf(path.sep));

    return gulp.src(stream)
      .pipe(merge({ fileName: fileName }))

      // since the question wanted a dist/lang/subfolders hierarchy
      .pipe(gulp.dest(path.join('./dist', 'lang', dirName)));
  });
});

// getFiles is recursive, if a "file" retrieved by fs.readdirSync is a directory

function getFiles(folders) {

  let possibleDirectory;

  folders.forEach(function (folder, index) {

    // read the file list from each directory
    let tempFiles = fs.readdirSync('./' + folder);

    tempFiles.forEach(function (fileOrDirectory) {

      possibleDirectory = path.join(folder, fileOrDirectory);
      if (fs.lstatSync(possibleDirectory).isDirectory())  {
        getFiles([possibleDirectory]);
      }
      else {

        // files[] will include all files found under the folders array
        files.push(path.join(folder, fileOrDirectory));

        if (baseNames.indexOf(fileOrDirectory) === -1) {

          // if that file name element doesn't already exist in baseName array
          // an array of just the basenames of true files
          baseNames.push(fileOrDirectory);
        }
      }
    });
  });
}

function makeStreams() {

  // for each file, find and save its parent directories without the folders[] "root" directories
  files.forEach(function (file) {

    let thisParentFolders = path.dirname(file).substr(file.indexOf(path.sep));

    if (parentFolders.indexOf(thisParentFolders) === -1) {

      // if that parentfolder  doesn't already exist in baseName array
      parentFolders.push(thisParentFolders);
    }
  });

  // now loop through all unique directories looking for those files with each parentFolder with baseName attached
  parentFolders.forEach(function (folder) {

    let foldersFile = folder.substr(folder.indexOf(path.sep));

    baseNames.forEach(function (baseName) {

      streams.push(files.filter(function (file) {
         return file.endsWith(path.join(foldersFile, baseName));
      }));
    });
  });

  //    Here: remove any "streams" (array sub-elements) that have only one file in them, length == 1
  //    But for now this filter is necessary due to a undefined entry in .
  streams = streams.filter( stream => stream.length >= 1);

  streams.forEach( (stream, index) => console.log("streams[" + index + "] = " + stream));
}