S Andrew
Reputation: 7298
How to group list of dict with condition in python
I have a below list of dict:
events_list =
[
{'attribute': '% Organic Matter - soil', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': True, 'ripe_mandated': False},
{'attribute': 'Cultivar', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': False, 'ripe_mandated': False},
{'attribute': 'Electrical conductivity', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': False, 'ripe_mandated': False},
{'attribute': 'Farm Acreage/tomato acreage', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': False, 'ripe_mandated': False},
{'attribute': 'Farm GPS coordinates', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': True, 'ripe_mandated': False},
{'attribute': 'Soil Amendment 1 date', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': True, 'ripe_mandated': False},
{'attribute': 'Air Temperature', 'event': 'Preharvest', 'stage': 'Preharvest', 'founder_mandated': True, 'ripe_mandated': False},
{'attribute': 'Activity Number', 'event': 'Shipping', 'stage': 'Transportation', 'founder_mandated': False, 'ripe_mandated': True},
{'attribute': 'Batch/Lot or Serial Number', 'event': 'Shipping', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
{'attribute': 'Data Owner', 'event': 'Shipping', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
{'attribute': 'Trading Partner', 'event': 'Shipping', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
{'attribute': 'Product Date', 'event': 'Shipping', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True}
{'attribute': 'Data Owner', 'event': 'Receiving', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
{'attribute': 'Product Date', 'event': 'Receiving', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
{'attribute': 'Product Identifier(s)', 'event': 'Receiving', 'stage': 'Transportation', 'founder_mandated': False, 'ripe_mandated': True},
{'attribute': 'Quantity', 'event': 'Receiving', 'stage': 'Transportation', 'founder_mandated': True, 'ripe_mandated': True},
]
In above list, we have 16 dict. In each dict, important keys are event and stage. I have to reformat the above data in such a way that all the attribute key belonging to same event and stage should come under 1 dict, and it should looks like below:
[
{
"event": "Preharvest",
"stage": "Preharvest",
"attributes": [
{
"attribute_name": "% Organic Matter - soil",
"founder_mandated": True,
"ripe_mandated": False
},
{
"attribute_name": "Cultivar",
"founder_mandated": False,
"ripe_mandated": False
},
# and so on for all attributes...
]
},
{
"event": "Shipping",
"stage": "Transportation",
"attributes": [
{
"attribute_name": "Batch/Lot or Serial Number",
"founder_mandated": True,
"ripe_mandated": True
},
{
"attribute_name": "Data Owner",
"founder_mandated": True,
"ripe_mandated": True
},
# and so on for all attributes...
]
},
{
"event": "Receiving",
"stage": "Transportation",
"attributes": [
{
"attribute_name": "Data Owner",
"founder_mandated": True,
"ripe_mandated": True
},
{
"attribute_name": "Product Date",
"founder_mandated": True,
"ripe_mandated": True
},
# and so on for all attributes...
]
}
]
So for this I have written below code:
first_time = True
new_events_list = []
for data in events_list:
new_event_dict = dict()
attr_dict = dict()
attr_list = []
found = False
if first_time:
new_event_dict['stage'] = data['stage']
new_event_dict['event'] = data['event']
attr_dict['attribute_name'] = data['attribute']
attr_dict['founder_mandated'] = data['founder_mandated']
attr_dict['ripe_mandated'] = data['ripe_mandated']
attr_list.append(attr_dict)
new_event_dict['attribute'] = attr_list
new_events_list.append(new_event_dict)
first_time = False
else:
for data2 in new_events_list:
if data['stage'] == data2['stage'] and data['event'] == data2['event']:
found = True
for data3 in data2['attribute']:
if data3['attribute_name'] != data['attribute']:
attr_dict['attribute_name'] = data['attribute']
attr_dict['founder_mandated'] = data['founder_mandated']
attr_dict['ripe_mandated'] = data['ripe_mandated']
data2['attribute'].append(attr_dict)
if not found:
new_event_dict['stage'] = data['stage']
new_event_dict['event'] = data['event']
attr_dict['attribute_name'] = data['attribute']
attr_dict['founder_mandated'] = data['founder_mandated']
attr_dict['ripe_mandated'] = data['ripe_mandated']
attr_list.append(attr_dict)
new_event_dict['attribute'] = attr_list
new_events_list.append(new_event_dict)
So the output of new_events_list looks like below:
[
{
"event": "Preharvest",
"stage": "Preharvest",
"attribute": [
{
"attribute_name": "% Organic Matter - soil",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Cultivar",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Electrical conductivity",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Electrical conductivity",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Farm Acreage/tomato acreage",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Farm Acreage/tomato acreage",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Farm Acreage/tomato acreage",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Farm Acreage/tomato acreage",
"founder_mandated": false,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Farm GPS coordinates",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Soil Amendment 1 date",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
},
{
"attribute_name": "Air Temperature",
"founder_mandated": true,
"ripe_mandated": false
}
]
},
{
"event": "Shipping",
"stage": "Transportation",
"attribute": [
{
"attribute_name": "Activity Number",
"founder_mandated": false,
"ripe_mandated": true
},
{
"attribute_name": "Batch/Lot or Serial Number",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Data Owner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Data Owner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Trading Partner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Trading Partner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Trading Partner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Trading Partner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
}
],
},
{
"event": "Receiving",
"stage": "Transportation",
"attribute": [
{
"attribute_name": "Data Owner",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Date",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Product Identifier(s)",
"founder_mandated": false,
"ripe_mandated": true
},
{
"attribute_name": "Product Identifier(s)",
"founder_mandated": false,
"ripe_mandated": true
},
{
"attribute_name": "Quantity",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Quantity",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Quantity",
"founder_mandated": true,
"ripe_mandated": true
},
{
"attribute_name": "Quantity",
"founder_mandated": true,
"ripe_mandated": true
}
]
}
]
But the problem is that the output has lots of duplicate attribute which we do not want. I have tried a lot but not able to figure a way to properly format the above data. Can anyone please help. Thanks
Upvotes: 1
Views: 53
Answers (2)
Mayank Porwal
Reputation: 34086
You can also use pandas for this:
In [454]: import pandas as pd
In [449]: df = pd.DataFrame(events_list)
In [453]: output = [{"event":x, "stage": y, "attribute":z.to_dict('records')} for (x,y),z in df.set_index(['event', 'stage']).groupby(['event', 'stage'])]
In [454]: output
Out[454]:
[{'event': 'Preharvest',
'stage': 'Preharvest',
'attribute': [{'attribute': '% Organic Matter - soil',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute': 'Cultivar',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute': 'Electrical conductivity',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute': 'Farm Acreage/tomato acreage',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute': 'Farm GPS coordinates',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute': 'Soil Amendment 1 date',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute': 'Air Temperature',
'founder_mandated': True,
'ripe_mandated': False}]},
{'event': 'Receiving',
'stage': 'Transportation',
'attribute': [{'attribute': 'Data Owner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute': 'Product Date',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute': 'Product Identifier(s)',
'founder_mandated': False,
'ripe_mandated': True},
{'attribute': 'Quantity',
'founder_mandated': True,
'ripe_mandated': True}]},
{'event': 'Shipping',
'stage': 'Transportation',
'attribute': [{'attribute': 'Activity Number',
'founder_mandated': False,
'ripe_mandated': True},
{'attribute': 'Batch/Lot or Serial Number',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute': 'Data Owner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute': 'Trading Partner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute': 'Product Date',
'founder_mandated': True,
'ripe_mandated': True}]}]
Upvotes: 0
Rakesh
Reputation: 82805
I believe you need.
res = {}
for i in events_list:
key = f"{i['event']}_{i['stage']}"
att = {'attribute_name': i.pop('attribute'), 'founder_mandated': i.pop('founder_mandated'), 'ripe_mandated': i.pop('ripe_mandated')}
if key not in res:
res[key] = {'attributes': [att], **i}
else:
res[key]['attributes'].append(att)
print(list(res.values()))
Output:
[{'attributes': [{'attribute_name': '% Organic Matter - soil',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute_name': 'Cultivar',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute_name': 'Electrical conductivity',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute_name': 'Farm Acreage/tomato acreage',
'founder_mandated': False,
'ripe_mandated': False},
{'attribute_name': 'Farm GPS coordinates',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute_name': 'Soil Amendment 1 date',
'founder_mandated': True,
'ripe_mandated': False},
{'attribute_name': 'Air Temperature',
'founder_mandated': True,
'ripe_mandated': False}],
'event': 'Preharvest',
'stage': 'Preharvest'},
{'attributes': [{'attribute_name': 'Activity Number',
'founder_mandated': False,
'ripe_mandated': True},
{'attribute_name': 'Batch/Lot or Serial Number',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute_name': 'Data Owner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute_name': 'Trading Partner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute_name': 'Product Date',
'founder_mandated': True,
'ripe_mandated': True}],
'event': 'Shipping',
'stage': 'Transportation'},
{'attributes': [{'attribute_name': 'Data Owner',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute_name': 'Product Date',
'founder_mandated': True,
'ripe_mandated': True},
{'attribute_name': 'Product Identifier(s)',
'founder_mandated': False,
'ripe_mandated': True},
{'attribute_name': 'Quantity',
'founder_mandated': True,
'ripe_mandated': True}],
'event': 'Receiving',
'stage': 'Transportation'}]
Upvotes: 2
Related Questions
- Group dicts from a list by the values of a certain key
- How to group a list of dicts based on part of the key from the dict
- PYTHON - group a list of dict
- Group list of dictionaries based on key
- Group dictionary items in a list by their specific keys
- Group python list of dictionaries by values
- How to do grouping of list of dictionary in Python?
- How to group dictionary keys based on their list values matching condition?
- Group the list of dictionary as list of values based on key
- How to group items in a list of dictionaries by matching values in python