CODEWITHSUNDEEP
javascriptjsonfs
Yasin
Yasin

Reputation: 11

Can't removes duplicates from json file

Here,I am trying to remove duplicates from my json file, so for that I tried to loop over data and remove repetitive from it. so for that I have written this code: `

var newData = [];
for (i = 0; i < playerData.length; i++) {
  for (j = 0; j < playerData[i].allTeams.length; j++) {
    for (k = 0; k < playerData[i].allTeams[j].listOfPlayer.length; k++) {
      for (s = k + 1; s < playerData[i].allTeams[j].listOfPlayer.length; s++) {
        if (
          playerData[i].allTeams[j].listOfPlayer[k].name ==
            playerData[i].allTeams[j].listOfPlayer[s].name &&
          playerData[i].allTeams[j].listOfPlayer[k].playerURL ==
            playerData[i].allTeams[j].listOfPlayer[s].playerURL
        ) {
          delete playerData[i].allTeams[j].listOfPlayer[s].name;
          delete playerData[i].allTeams[j].listOfPlayer[s].playerRole;
          delete playerData[i].allTeams[j].listOfPlayer[s].playerURL;
        }
      }
    }
  }
  newData.push(playerData[i]);
}`

first of all I tried to use this code in my sample data and it successfully work, but when I run this code for huge amount of data which is in file main.json then it is not working. what stopping me from this??

here is link of the the code and data: https://codesandbox.io/s/flamboyant-dewdney-40jw5?file=/src/index.js

Upvotes: 0

Views: 91

Answers (3)

Kinglish
Kinglish

Reputation: 23664

Try using map() to iterate and .filter() to filter out duplicates. Here is a testable snippet using your sample.json data

const playerData = [{
  "clubName": "ADSK Cricket Club",
  "clubUrl": "https://cricclubs.com/baca/viewInternalClub.do?internalClubId=32&clubId=1755",
  "allTeams": [{
      "teamName": "Afgan Zalmi",
      "teamURL": "https://cricclubs.com/baca/viewTeam.do?teamId=393&clubId=1755",
      "listOfPlayer": []
    },
    {
      "teamName": "Autodesk CC",
      "teamURL": "https://cricclubs.com/baca/viewTeam.do?teamId=376&clubId=1755",
      "listOfPlayer": [{
          "name": "AMARNATH AMARNATH",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=893120&clubId=1755"
        },
        {
          "name": "AMARNATH AMARNATH",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=893120&clubId=1755"
        },
        {
          "name": "AMITH RAUL",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=885821&clubId=1755"
        },
        {
          "name": "ASHISH ASHISH",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=893119&clubId=1755"
        },
        {
          "name": "HARPREET KHAIRA",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=885820&clubId=1755"
        },
        {
          "name": "HEMANT MURARKA",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=885822&clubId=1755"
        },
        {
          "name": "HERSHAL K",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=943445&clubId=1755"
        },
        {
          "name": "VISHAL PHIRANI",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=885851&clubId=1755"
        },
        {
          "name": "IMRAN SYED",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=839822&clubId=1755"
        },
        {
          "name": "VISHAL PHIRANI",
          "playerRole": " All Rounder ",
          "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=885851&clubId=1755"
        }
      ]
    }
  ]
}]
const nplayerData = playerData.map( set => {
set.allTeams = set.allTeams.map( p => {
p.listOfPlayer = p.listOfPlayer.filter((t, i, s) =>
    i === s.findIndex((pl) => (
      pl.name === t.name && pl.playerURL.trim() === t.playerURL.trim()
    ))
  );
return p;
})
return set
})
console.log(nplayerData);

Upvotes: 0

messias junio
messias junio

Reputation: 11

Try this both functions:

getUniqueListBy(arr, key): Unique values from of objects, and you can specify the key

getRemoveNestedDuplicates(): Traverse the Nested values

const playerData = [
  {
    "clubName": "ADSK Cricket Club",
    "clubUrl": "https://cricclubs.com/baca/viewInternalClub.do?internalClubId=32&clubId=1755",
    "allTeams": [
      {
        "teamName": "Afgan Zalmi",
        "teamURL": "https://cricclubs.com/baca/viewTeam.do?teamId=393&clubId=1755",
        "listOfPlayer": []
      },
      {
        "teamName": "Autodesk CC",
        "teamURL": "https://cricclubs.com/baca/viewTeam.do?teamId=376&clubId=1755",
        "listOfPlayer": [
          {
            "name": "AMARNATH AMARNATH",
            "playerRole": " All Rounder ",
            "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=893120&clubId=1755"
          },
          {
            "name": "AMARNATH AMARNATH",
            "playerRole": " All Rounder ",
            "playerURL": "https://cricclubs.com/baca/viewPlayer.do?playerId=893120&clubId=1755"
          }
        ]
      }
    ]
  }
]


function getUniqueListBy(arr, key) {
  if (Array.isArray(arr)) {
    return [...new Map(arr.map(item => [item[key], item])).values()]
  }
  return []
}

function getRemoveNestedDuplicates(data) {
  for (prop in data) {
    if (typeof(data[prop]) == 'object') {
      if (data[prop] && data[prop].listOfPlayer) {
        data[prop].listOfPlayer = getUniqueListBy(data[prop].listOfPlayer, 'name')
      } 
      getRemoveNestedDuplicates(data[prop])
    }
  }
}

getRemoveNestedDuplicates(playerData)


for (const data of playerData) {
  console.log(data.allTeams[1].listOfPlayer)
}

Upvotes: 0

Raky
Raky

Reputation: 900

Please replace your json data in the js variable players. This should work for you.

var newData = [];

function removeDuplicates() {

  // Create an array of objects
  players = [{
      name: "James",
      playerrole: "Bjarne",
      playerURL:"Captain"
    },
    {
     name: "James",
      playerrole: "Bjarne",
      playerURL:"Captain"
    },
    {
      name: "James",
      playerrole: "Bjarne",
      playerURL:"Captain"
    },
    {
      name: "James",
      playerrole: "Bjarne",
      playerURL:"Captain"
    }
  ];

  jsonObject = players.map(JSON.stringify);
  uniqueSet = new Set(jsonObject);
  newData = Array.from(uniqueSet).map(JSON.parse);

  console.log(newData);

}
<p>
  Click on the button to remove the duplicates in the array
</p>

<p>Check the console for the output</p>

<button onclick="removeDuplicates()">
  Click here
</button>

Upvotes: 1

Related Questions