Recursively remove None value or None key from nested dict

Question

Question:

How can I iterate over a dictionary and remove None keys or values from it?

Here is what I've tried:

Code:

import copy


def _ignore(data):
    copied_data = copy.deepcopy(data)

    print('-------------------------')
    print(f'copied_data: {copied_data}')
    print('-------------------------')

    if isinstance(copied_data, list):
        print(f'item is instance of list: {copied_data}')

        for idx, item in enumerate(data):
            if isinstance(item, list):
                return _ignore(item)

            elif isinstance(item, dict):
                return _ignore(item)

            elif item is None:
                del copied_data[idx]

    elif isinstance(copied_data, dict):
        print(f'item is instance of dict: {copied_data}')

        for key, item in data.items():
            if isinstance(item, list):
                return _ignore(item)

            elif isinstance(item, dict):
                return _ignore(item)

            elif item is None:
                del copied_data[key]

    return copied_data


if __name__ == '__main__':
    data = {'key1': None, 'key2': {'k1': None, 'k2': 2}, 'key3': [None, 1, 2, 3, 4]}

    print(f'output: {_ignore(data=data)}')

Input:

{'key1': None, 'key2': {'k1': None, 'k2': 2}, 'key3': [None, 1, 2, 3, 4]}

Output:

-------------------------
copied_data: {'key1': None, 'key2': {'k1': None, 'k2': 2}, 'key3': [None, 1, 2, 3, 4]}
-------------------------
item is instance of dict: {'key1': None, 'key2': {'k1': None, 'k2': 2}, 'key3': [None, 1, 2, 3, 4]}
-------------------------
copied_data: {'k1': None, 'k2': 2}
-------------------------
item is instance of dict: {'k1': None, 'k2': 2}
output: {'k2': 2}

Keep in mind that the code also should remove None values in a nested list or nested dict and we can't remove an element from dict while we are iterating over it.

Update: The code should support nested dict or list

Here is another input sample:

{'key1': None, None:1 ,'key2': {'k1': {'k3': [None, 1, 23], 'k4': None}, 'k2': 2},
        'key3': [{'key1': None, 'key2': [None, 1, 2, 3], 'key3': {'k1': 1}}, 1, 2, 3, 4]} 

Thanks.

Answer 1

Here is one way to do it by creating new objects.

def removed_none(obj):
    t = type(obj)
    if issubclass(t, (tuple, list, set)):
        obj = t(removed_none(a) for a in obj if a is not None)
    elif issubclass(t, dict):
        obj = {k: removed_none(v) for k, v in obj.items()
               if k is not None and v is not None}
    return obj

data = {'key1': None, 'key2': {'k1': None, 'k2': {1: 2, 3: None}},
        'key3': [None, [1, None, 0], 2, 3, 4]}
print(f'output: {removed_none(data)}')
# output: {'key2': {'k2': {1: 2}}, 'key3': [[1, 0], 2, 3, 4]}

If you want to modify the existing object in place, here is a different function (remove_none instead of removed_none). Note this cannot be use to modify tuples containing None values, as these are immutable.

def remove_none(obj):
    if isinstance(obj, list):
        for i in range(len(obj) - 1, -1, -1):
            if obj[i] is None:
                del obj[i]
            else:
                remove_none(obj[i])
    elif isinstance(obj, set):
        obj.discard(None)
        for elem in obj:
            remove_none(elem)
    elif isinstance(obj, dict):
        for k, v in list(obj.items()):
            if k is None or v is None:
                del obj[k]
            else:
                remove_none(v)

data = {'key1': None, 'key2': {'k1': None, 'k2': {1: 2, 3: None}},
        'key3': [None, [1, None, 0], 2, 3, 4]}
remove_none(data)
print(f'output: {data}')
# output: {'key2': {'k2': {1: 2}}, 'key3': [[1, 0], 2, 3, 4]}

Answer 2

You're not doing what you're claiming to do, since you're only removing None values as far as I can see. This dict:

{'key1': None, 'key2': {'k1': None, 'k2': 2}, 'key3': [None, 1, 2, 3, 4]}

Only has None values, not keys. I'm gonna insist on the Python definition of keys and values here. A None key looks like:

{None: "value"}

However, you could easily do it something like this:

def recursive_filter(item, *forbidden):
    if isinstance(item, list):
        return [recursive_filter(entry, *forbidden) for entry in item if entry not in forbidden]
    if isinstance(item, dict):
        result = {}
        for key, value in item.items():
            value = recursive_filter(value, *forbidden)
            if key not in forbidden and value not in forbidden:
                result[key] = value
        return result
    return item

Which you can use like:

clean = recursive_filter(dirty, None)

Or if you want to filter out more:

clean = recursive_filter(dirty, *iterable_of_forbidden_things)
clean = recursive_filter(dirty, None, other_forbidden_thing)

If you really only care about values, then you can just remove the checks about the keys.

Answer 3

You could recursively check and return a key, value pair, and then create a new dict from it like,

$ cat rmnone.py
data = {'key1': None, 'key2': {'k1': {'k3': [None, 1, 23], 'k4': None}, 'k2': 2},
        'key3': [{'key1': None, 'key2': [None, 1, 2, 3], 'key3': {'k1': 1}}, 1, 2, 3, 4]}

def func(data):
    for key, val in data.items():
        if val is not None:
            if isinstance(val, list):
                tmp = []
                for v in val:
                    if isinstance(v, dict):
                        for k,v in func(v):
                            tmp.append({k:v})
                    elif v is not None:
                        tmp.append(v)
                yield (key, tmp)
            if isinstance(val, tuple):
                tmp = []
                for v in val:
                    if isinstance(v, dict):
                        for k,v in func(v):
                            tmp.append({k:v})
                    elif v is not None:
                        tmp.append(v)
                yield (key, tmp)
            elif isinstance(val, dict):
                    tmp = {}
                    for k,v in func(val):
                        tmp.update({k:v})
                    yield key, tmp
            elif not isinstance(val, (tuple, dict, list)):
                yield key,val

d = {k:v for k,v in func(data)}
print(d)

Output:

$ python3 rmnone.py
{'key2': {'k1': {'k3': [1, 23]}, 'k2': 2}, 'key3': [{'key2': [1, 2, 3]}, {'key3': {'k1': 1}}, 1, 2, 3, 4]}