CODEWITHSUNDEEP
pythonrecursion
user10332687
user10332687

Reputation:

Flattening a dict

I have the following array of dicts (there's only one dict):

[{
    'RuntimeInMinutes': '21',
    'EpisodeNumber': '21',
    'Genres': ['Animation'],
    'ReleaseDate': '2005-02-05',
    'LanguageOfMetadata': 'EN',
    'Languages': [{
        '_Key': 'CC',
        'Value': ['en']
    }, {
        '_Key': 'Primary',
        'Value': ['EN']
    }],
    'Products': [{
        'URL': 'http://www.hulu.com/watch/217566',
        'Rating': 'TV-Y',
        'Currency': 'USD',
        'SUBSCRIPTION': '0.00',
        '_Key': 'US'
    }, {
        'URL': 'http://www.hulu.com/d/217566',
        'Rating': 'TV-Y',
        'Currency': 'USD',
        'SUBSCRIPTION': '0.00',
        '_Key': 'DE'
    }],
    'ReleaseYear': '2005',
    'TVSeriesID': '5638#TVSeries',
    'Type': 'TVEpisode',
    'Studio': '4K Media'
}]

I would like to flatten the dict as follows:

[{
    'RuntimeInMinutes': '21',
    'EpisodeNumber': '21',
    'Genres': ['Animation'],
    'ReleaseDate': '2005-02-05',
    'LanguageOfMetadata': 'EN',
    'Languages._Key': ['CC', 'Primary'],
    'Languages.Value': ['en', 'EN'],
    'Products.URL': ['http://www.hulu.com/watch/217566', 'http://www.hulu.com/d/217566'],
    'Products.Rating': ['TV-Y', 'TV-Y'],
    'Products.Currency': ['USD', 'USD'],
    'Products.SUBSCRIPTION': ['0.00', '0.00'],
    'Products._Key': ['US', 'DE'],
    'ReleaseYear': '2005',
    'TVSeriesID': '5638#TVSeries',
    'Type': 'TVEpisode',
    'Studio': '4K Media'
}]

In other words, anytime a dict is encountered, it need to convert to either a string, number, or list.

What I currently have is something along the lines of the following, which uses a while loop to iterate through all the subpaths of the json.

    while True:

        for key in copy(keys):

            val = get_sub_object_from_path(obj, key)

            if isinstance(val, dict):
                FLAT_OBJ[key.replace('/', '.')] = val
            else:
                keys.extend(os.path.join(key, _nextkey) for _nextkey in val.keys())
            keys.remove(key)

        if (not keys) or (n > 5):
            break
        else:
            n += 1
            continue

Upvotes: 1

Views: 229

Answers (3)

panda-34
panda-34

Reputation: 4209

Ajax1234's answer loses values of 'Genres' and 'Languages.Value' Here's a bit more generic version:

def flatten_obj(data):
    def flatten_item(item, keys):
        if isinstance(item, list):
            for v in item:
                yield from flatten_item(v, keys)
        elif isinstance(item, dict):
            for k, v in item.items():
                yield from flatten_item(v, keys+[k])
        else:
            yield '.'.join(keys), item

    res = []
    for item in data:
        res_item = defaultdict(list)
        for k, v in flatten_item(item, []):
            res_item[k].append(v)
        res.append({k: (v if len(v) > 1 else v[0]) for k, v in res_item.items()})
    return res

P.S. "Genres" value is also flattened. It is either an inconsistency in the OP requirements or a separate problem which is not addressed in this answer.

Upvotes: 0

cdlane
cdlane

Reputation: 41872

EDIT

This now appears to be fixed:

As @panda-34 correctly points out (+1), the currently accepted solution loses data, specifically Genres and Languages.Value when you run the posted code.

Unfortunately, @panda-34's code modifies Genres:

'Genres': 'Animation',

rather than leaving it alone as in the OP's example:

'Genres': ['Animation'],

Below's my solution which attacks the problem a different way. None of the keys in the original data contains a dictionary as a value, only non-containers or lists (e.g. lists of dictionaries). So a primary a list of dictionaries will becomes a dictionary of lists (or just a plain dictionary if there's only one dictionary in the list.) Once we've done that, then any value that's now a dictionary is expanded back into the original data structure:

def flatten(container):
    # A list of dictionaries becomes a dictionary of lists (unless only one dictionary in list)
    if isinstance(container, list) and all(isinstance(element, dict) for element in container):
        new_dictionary = {}

        first, *rest = container

        for key, value in first.items():
            new_dictionary[key] = [flatten(value)] if rest else flatten(value)

        for dictionary in rest:
            for key, value in dictionary.items():
                new_dictionary[key].append(value)

        container = new_dictionary

    # Any dictionary value that's a dictionary is expanded into original dictionary
    if isinstance(container, dict):
        new_dictionary = {}

        for key, value in container.items():
            if isinstance(value, dict):
                for sub_key, sub_value in value.items():
                    new_dictionary[key + "." + sub_key] = sub_value
            else:
                new_dictionary[key] = value

        container = new_dictionary

    return container

OUTPUT

{
    "RuntimeInMinutes": "21",
    "EpisodeNumber": "21",
    "Genres": [
        "Animation"
    ],
    "ReleaseDate": "2005-02-05",
    "LanguageOfMetadata": "EN",
    "Languages._Key": [
        "CC",
        "Primary"
    ],
    "Languages.Value": [
        [
            "en"
        ],
        [
            "EN"
        ]
    ],
    "Products.URL": [
        "http://www.hulu.com/watch/217566",
        "http://www.hulu.com/d/217566"
    ],
    "Products.Rating": [
        "TV-Y",
        "TV-Y"
    ],
    "Products.Currency": [
        "USD",
        "USD"
    ],
    "Products.SUBSCRIPTION": [
        "0.00",
        "0.00"
    ],
    "Products._Key": [
        "US",
        "DE"
    ],
    "ReleaseYear": "2005",
    "TVSeriesID": "5638#TVSeries",
    "Type": "TVEpisode",
    "Studio": "4K Media"
}

But this solution introduces a new apparent inconsistency:

'Languages.Value': ['en', 'EN'],

vs.

"Languages.Value": [["en"], ["EN"]],

However, I believe this is tied up with the Genres inconsistency mentioned earlier and the OP needs to define a consistent resolution.

Upvotes: 1

Ajax1234
Ajax1234

Reputation: 71451

You can use recursion with a generator:

from collections import defaultdict
_d = [{'RuntimeInMinutes': '21', 'EpisodeNumber': '21', 'Genres': ['Animation'], 'ReleaseDate': '2005-02-05', 'LanguageOfMetadata': 'EN', 'Languages': [{'_Key': 'CC', 'Value': ['en']}, {'_Key': 'Primary', 'Value': ['EN']}], 'Products': [{'URL': 'http://www.hulu.com/watch/217566', 'Rating': 'TV-Y', 'Currency': 'USD', 'SUBSCRIPTION': '0.00', '_Key': 'US'}, {'URL': 'http://www.hulu.com/d/217566', 'Rating': 'TV-Y', 'Currency': 'USD', 'SUBSCRIPTION': '0.00', '_Key': 'DE'}], 'ReleaseYear': '2005', 'TVSeriesID': '5638#TVSeries', 'Type': 'TVEpisode', 'Studio': '4K Media'}]

def get_vals(d, _path = []):
  for a, b in getattr(d, 'items', lambda :{})():
    if isinstance(b, list) and all(isinstance(i, dict) or isinstance(i, list) for i in b):
       for c in b:
         yield from get_vals(c, _path+[a])
    elif isinstance(b, dict):
       yield from get_vals(b, _path+[a])
    else:
       yield ['.'.join(_path+[a]), b]

results = [i for b in _d for i in get_vals(b)]
_c = defaultdict(list)
for a, b in results:
  _c[a].append(b)

result = [{a:list(b) if len(b) > 1 else b[0] for a, b in _c.items()}]
import json
print(json.dumps(result, indent=4))

Output:

[
  {
    "RuntimeInMinutes": "21",
    "EpisodeNumber": "21",
    "Genres": [
        "Animation"
    ],
    "ReleaseDate": "2005-02-05",
    "LanguageOfMetadata": "EN",
    "Languages._Key": [
        "CC",
        "Primary"
    ],
    "Languages.Value": [
        [
            "en"
        ],
        [
            "EN"
        ]
    ],
    "Products.URL": [
        "http://www.hulu.com/watch/217566",
        "http://www.hulu.com/d/217566"
    ],
    "Products.Rating": [
        "TV-Y",
        "TV-Y"
    ],
    "Products.Currency": [
        "USD",
        "USD"
    ],
    "Products.SUBSCRIPTION": [
        "0.00",
        "0.00"
    ],
    "Products._Key": [
        "US",
        "DE"
    ],
    "ReleaseYear": "2005",
    "TVSeriesID": "5638#TVSeries",
    "Type": "TVEpisode",
    "Studio": "4K Media"
  }
]

Edit: wrapping solution in outer function:

def flatten_obj(data):
  def get_vals(d, _path = []):
    for a, b in getattr(d, 'items', lambda :{})():
      if isinstance(b, list) and all(isinstance(i, dict) or isinstance(i, list) for i in b):
        for c in b:
          yield from get_vals(c, _path+[a])
      elif isinstance(b, dict):
        yield from get_vals(b, _path+[a])
      else:
        yield ['.'.join(_path+[a]), b]
  results = [i for b in data for i in get_vals(b)]
  _c = defaultdict(list)
  for a, b in results:
     _c[a].append(b)
  return [{a:list(b) if len(b) > 1 else b[0] for a, b in _c.items()}]

Upvotes: 5

Related Questions