How to remove duplicate objects from JavaScript array

Question

This is my code

var studentsList = [
    {"Id": "101", "name": "siva"},
    {"Id": "101", "name": "siva"},
    {"Id": "102", "name": "hari"},
    {"Id": "103", "name": "rajesh"},
    {"Id": "103", "name": "rajesh"},
    {"Id": "104", "name": "ramesh"},
];

function arrUnique(arr) {
    var cleaned = [];
    studentsList.forEach(function(itm) {
        var unique = true;
        cleaned.forEach(function(itm2) {
            if (_.isEqual(itm, itm2)) unique = false;
        });
        if (unique)  cleaned.push(itm);
    });
    return cleaned;
}

var uniqueStandards = arrUnique(studentsList);

document.body.innerHTML = '<pre>' + JSON.stringify(uniqueStandards, null, 4) + '</pre>';

OutPut

[
{
    "Id": "101",
    "name": "siva"
},
{
    "Id": "102",
    "name": "hari"
},
{
    "Id": "103",
    "name": "rajesh"
},
{
    "Id": "104",
    "name": "ramesh"
}
]

In the above code I got unique objects from the JavaScript array, but i want to remove duplicate objects. So I want to get without duplicate objects from the array, the output like [ { "Id": "102", "name": "hari" }, { "Id": "104", "name": "ramesh" } ]

Answer 1

How about this, two different examples, the first gets all duplicates comparing a property rather than an Id or comparing the whole object. The second example is is a distinct list of duplicates.

// Sample 1 - Show all duplicates of a particular code rather than the whole record is unique

var sampleJson = 
[
  { id: 1, code: "123-123" }, 
  { id: 2, code: "123-124" }, 
  { id: 3, code: "123-125" }, 
  { id: 4, code: "123-123" }
];

console.log("All duplicates");

let duplicates= [];

sampleJson.forEach(function (item) {
  var checkForDuplicates = sampleJson.filter(a => a.code == item.code);
  if (checkForDuplicates.length > 1) {
    duplicates .push(item);
  }
});
console.log(duplicates);
console.log( "Distinct duplicates");


// Sample 2 - Distinct list of duplicates
let distinctDuplicates= [];

sampleJson.forEach(function(item) {

  var checkForDuplicates = sampleJson.filter(a => a.code == item.code);
  if (checkForDuplicates.length > 1) {

    // Ensure we only have the single duplicate by ensuring we do not add the same one twice
    var exists = distinctDuplicates.filter(b => b.code == item.code).length == 1;
    if (false === exists) {
      distinctDuplicates.push(item);
    }
  }
});
console.log(distinctDuplicates);

Answer 2

How about this one-liner function using Set()?

function arrUnique(arr) { return [...new Set(arr.map(JSON.stringify))].map(JSON.parse);}

It uses JSON.stringify() and JSON.parse() to transform the objects to string, because the equality function of Set() (===) is not customizable and doesn't work for objects, but for strings. JSON.parse() of the unique elements in the stringified version transforms them back to objects.

Very likely, this is not the fastest version because of overhead using JSON methods.

Answer 3

You can use library lodash and method uniqBy()

Answer 4

var unique = arr.filter(function(elem, index, self) {
    return index === self.indexOf(elem);
})

Answer 5

This maybe? (not the most performant implementation but gets the job done):

var studentsList = [
  {Id: "101", name: "siva"},
  {Id: "101", name: "siva"},
  {Id: "102", name: "hari"},
  {Id: "103", name: "rajesh"},
  {Id: "103", name: "rajesh"},
  {Id: "104", name: "ramesh"},
];

var ids = {};

studentsList.forEach(function (student) {
  ids[student.Id] = (ids[student.Id] || 0) + 1;
});

var output = [];
studentsList.forEach(function (student) {
  if (ids[student.Id] === 1) output.push(student);
});

console.log(output);

Edit: faster method if the students are ordered by Id:

var studentsList = [
  {Id: "101", name: "siva"},
  {Id: "101", name: "siva"},
  {Id: "102", name: "hari"},
  {Id: "103", name: "rajesh"},
  {Id: "103", name: "rajesh"},
  {Id: "104", name: "ramesh"},
];

var output = [];
studentsList.reduce(function (isDup, student, index) {
  var nextStudent = studentsList[index + 1];
  if (nextStudent && student.Id === nextStudent.Id) {
    return true;
  } else if (isDup) {
    return false;
  } else {
    output.push(student);
  }

  return false;
}, false);

console.log(output);

Answer 6

Here is the controller which will detect the duplicate element and remove it and will give you the element which has no duplicates. Just use this controller

$scope.names= studentsList.reduce(function(array, place) {
         if (array.indexOf( studentsList.name) < 0) 
                array.push( studentsList.name );
         else
                array.splice(array.indexOf( studentsList.name), 1);
         return array;
}, []);

Hope this works for your case

Answer 7

Check this

    var uniqueStandards = UniqueArraybyId(studentsList ,"id");

    function UniqueArraybyId(collection, keyname) {
              var output = [], 
                  keys = [];

              angular.forEach(collection, function(item) {
                  var key = item[keyname];
                  if(keys.indexOf(key) === -1) {
                      keys.push(key);
                      output.push(item);
                  }
              });
        return output;
   };

Answer 8

You can use javascript's filter method:

The solution for the above problem can be found in this fiddle. The performance of this solution will be better because we are using pure javascript and there is no third party overhead.

app.controller('myCntrl',function($scope){

    var seen = {};
    //You can filter based on Id or Name based on the requirement
    var uniqueStudents = studentsList.filter(function(item){
    if(seen.hasOwnProperty(item.Id)){
        return false;
    }else{
        seen[item.Id] = true;
        return true;
    }
});
    $scope.students = uniqueStudents;

});

Let me know if you need any other details.