javascript - Remove exact duplicate row from array

Question

I have a following list in Javascript.

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

This list have many duplicate rows. I want to remove all the duplicate rows out of it.

The object can have more 10 properties also, so checking each property is not the solution. Plus whatever way I got online are either for 1 dimensional array or to remove duplicates by specific property. How can I maintain a list containing all unique rows?

Answer 1

You could use a hash table with only the sorted keys as hash and an array for the objects with the same keys.

{
    "EXECUTE|ID|NAME|TYPE": [
        { ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
        { ID: "1234", /* ... */ },
        { ID: "RI0031", /* ... */ },
        { ID: "RI04", /* ... */ },
        { ID: "RID005", /* ... */ }
    ],
    "EXECUTE|NAME|TYPE": [
        { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" },
        { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" },
        { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }
    ]
}

For filtering, you could first check the keys, joined with pipe, in the hash table and then if found, iterate the array of the hash table and check all properties. If not equal, then push the actual object to the hash table, and return true.

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter(function (hash) {
        return function (o) {
            var keys = Object.keys(o).sort(),
                key = keys.join('|');

            if (!hash[key]) {
                hash[key] = [o];
                return true;
            }
            if (!hash[key].some(function (p) { return keys.every(function (k) { return o[k] === p[k]; }); })) {
                hash[key].push(o);
                return true;
            }
        };
    }(Object.create(null)));

console.log(result);

.as-console-wrapper { max-height: 100% !important; top: 0; }

ES6 with Map

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter((map => o => {
        var keys = Object.keys(o).sort(),
            key = keys.join('|');

        if (!map.has(key)) {
            map.set(key, [o]);
            return true;
        }
        if (!map.get(key).some(p => keys.every(k => o[k] === p[k]))) {
            map.get(key).push(o);
            return true;
        }
    })(new Map));

console.log(result);

.as-console-wrapper { max-height: 100% !important; top: 0; }

Answer 2

The following snippet will remove the duplicates by constructing a hash map with JSON stringified keys and the corresponding object items as values. By not stringifying the values, we keep their references. So no JSON.parse() operation is used here.

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? h : (h[os] = o, h);
  },{});
  return Object.keys(hash)
               .map(k => hash[k]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

However if you would like to remove all occurrences of a duplicated item then you may still do the job in a very similar fashion like;

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? (h[os] = false,h) : (h[os] = o, h);
                      },{});
  return Object.keys(hash)
               .reduce((r,k) => hash[k] === false ? r : r.concat(hash[k]),[]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

Answer 3

From the MDN article on JSON.stringify:

Properties of non-array objects are not guaranteed to be stringified in any particular order. Do not rely on ordering of properties within the same object within the stringification.

Instead of comparing the JSON stringification of each object directly, one should first use Object.entries to get an array of [key, value] pairs and sort it by the string value of key. This sorted entries array can then be safely stringified and compared regardless of the JSON.stringify implementation.

Note that my example below uses the latest ES6 features, including Set and Object.entries. To transpile this code to ES5 and run it, please visit https://babeljs.io/repl/.

function deepUnique (array) {
  return [...new Set(array.map(e => JSON.stringify(Object.entries(e).sort(compareEntriesByKey))))].map(e => objectFromEntries(JSON.parse(e)))
}

function compareEntriesByKey (a, b) {
  return a[0] < b[0] ? -1 : a[0] === b[0] ? 0 : 1
}

function objectFromEntries (entries) {
  let result = {}
  entries.forEach(e => result[e[0]] = e[1])
  return result
}

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

console.log(deepUnique(list))

Credit to Kind user for creating an answer using both JSON and Set before I posted mine.

Answer 4

Stringifies all elements, while checking if similar ones exist and then parses them. JSON.stringify() and JSON.parse() are not incredibly fast, though. Keep that in mind, if your objects are quite large.

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

function removeDuplicates(list) {
  return list.reduce(function(memo, obj) {
    var string = JSON.stringify(obj);

    if (memo.indexOf(string) === -1) {
      memo.push(string);
    }

    return memo;
  }, []).map(function(stringified) {
    return JSON.parse(stringified);
  });
}

list = removeDuplicates(list);
console.log(list);

Answer 5

This will remove exact duplicates from your array of objects.

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
     { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"},
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}, 
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {ID: "RID005",NAME: "CALCULATE FAST GROUP",EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"}, 
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}];
     
var filtered = [...new Set(list.map(v => JSON.stringify(v)))];

console.log(filtered.map(v => JSON.parse(v)));