Exclude empty/null values from JSON serialization

Question

I am serializing multiple nested dictionaries to JSON using Python with simplejson.

Is there any way to automatically exclude empty/null values?

For example, serialize this:

 {
     "dict1" : {
     "key1" : "value1",
     "key2" : None
     }
 }

to

 {
     "dict1" : {
     "key1" : "value1"
     }
 }

When using Jackson with Java you can use Inclusion.NON_NULL to do this. Is there a simplejson equivalent?

Answer 1

I used the MatanRubin function and extended it so that it also filters Nan (float) and Null (string) to be able to talk with a Php API.

from math import isnan

def clean_nones(value):
    """
    Recursively remove all None values from dictionaries and lists, and returns
    the result as a new dictionary or list.
    """
    def checkNan(value):
        if isinstance(value, float) and isnan(value):
            return True if (isinstance(value, float) and isnan(value)) else False
        
    if isinstance(value, list):
        return [clean_nones(x) for x in value if (x is not None and  x != 'NULL' and not checkNan(x))]
    elif isinstance(value, dict):
        return {
            key: clean_nones(val)
            for key, val in value.items()
            if (val is not None and val != 'NULL' and not checkNan(val))
        }
    else:
        return value

cleanedJson = clean_nones(toCleanJson)

Answer 2

This solution is correction of the one above from @eric which does not handle list type corectly.

Values in canonical JSON dictionary can be of one of following 3 types:

• dictionary
• list
• value type (string, integer or floating point)

Note: Assumption is that we are dealing here with canonical JSON dictionary which can really contain only above mentioned types. If dictionary contains other types then ones mentioned above (e.g. tuples, custom classes, ...), then this solution won't work as expected.

The essential difference between this solution (below) and the original one from @eric is that list can contain elements of dictionary type from iside of which we want to drop elements with None value.

def cleandict(d):
    if isinstance(d, dict):
        return {k: cleandict(v) for k, v in d.items() if v is not None}
    elif isinstance(d, list):
        return [cleandict(v) for v in d]
    else:
        return d

Note: Please keep in mind that we must NOT remove None elements from the list since it would affect structural integrity of the list data. If some ( or all) of list elements have None value, they shall remain listed in the list structure as they were in order to preserve original structural meaning/integrity of the list.

Answer 3

Could you maybe remain 'url' if it has value in one place and remove it if it none on another place?

'inline_keyboard': [
        [
            {'text': '0-0', 'url': 'someValue', 'login_url': None, 'callback_data': '0-0', 'switch_inline_query': None},
            {'text': '0-1', 'url': None, 'login_url': None, 'callback_data': '0-1', 'switch_inline_query': None}
        ],
        [
            {'text': '1-0', 'url': None, 'login_url': None, 'callback_data': '1-0', 'switch_inline_query': None},
            {'text': '1-1', 'url': None, 'login_url': None, 'callback_data': '1-1', 'switch_inline_query': None}
        ],
        [
            {'text': '2-0', 'url': None, 'login_url': None, 'callback_data': '2-0', 'switch_inline_query': None}
        ]
]

Answer 4

You can try this approach. In my case (I use python 3), it works well.

def to_json(self):
    return json.dumps(self,
                      default=lambda o: dict((key, value) for key, value in o.__dict__.items() if value),
                      indent=4,
                      allow_nan=False)

Answer 5

It works for me:

When dictionary has dict/list/tuple values ....

for example it is my object:

dict_obj = {
    'inline_keyboard': [
        [
            {'text': '0-0', 'url': None, 'login_url': None, 'callback_data': '0-0', 'switch_inline_query': None},
            {'text': '0-1', 'url': None, 'login_url': None, 'callback_data': '0-1', 'switch_inline_query': None}
        ],
        [
            {'text': '1-0', 'url': None, 'login_url': None, 'callback_data': '1-0', 'switch_inline_query': None},
            {'text': '1-1', 'url': None, 'login_url': None, 'callback_data': '1-1', 'switch_inline_query': None}
        ],
        [
            {'text': '2-0', 'url': None, 'login_url': None, 'callback_data': '2-0', 'switch_inline_query': None}
        ]
    ]
}

I wrote this function:

def delete_none_values(obj):
    if isinstance(obj, dict):
        for k, v in list(obj.items()):
            if v is None:
                del obj[k]
            elif isinstance(v, dict):
                delete_none_values(v)
            elif isinstance(v, (list, tuple)):
                for _ in v:
                    delete_none_values(_)
    elif isinstance(obj, (list, tuple)):
        for _ in obj:
            delete_none_values(_)
    return obj

And then when use this fuction:

from json import dumps

print(
    dumps(
        delete_none_values(dict_obj.copy()),
        indent=2
    )
)

output is:

{
  "inline_keyboard": [
    [
      {"text": "0-0", "callback_data": "0-0"},
      {"text": "0-1", "callback_data": "0-1"}
    ],
    [
      {"text": "1-0", "callback_data": "1-0"},
      {"text": "1-1", "callback_data": "1-1"}
    ],
    [
      {"text": "2-0", "callback_data": "2-0"}
    ]
  ]
}

Answer 6

My Python3 version of this has the benefit of not changing the input, as well as recursion into dictionaries nested in lists:

def clean_nones(value):
    """
    Recursively remove all None values from dictionaries and lists, and returns
    the result as a new dictionary or list.
    """
    if isinstance(value, list):
        return [clean_nones(x) for x in value if x is not None]
    elif isinstance(value, dict):
        return {
            key: clean_nones(val)
            for key, val in value.items()
            if val is not None
        }
    else:
        return value

For example:

a = {
    "a": None,
    "b": "notNone",
    "c": ["hello", None, "goodbye"],
    "d": [
        {
            "a": "notNone",
            "b": None,
            "c": ["hello", None, "goodbye"],
        },
        {
            "a": "notNone",
            "b": None,
            "c": ["hello", None, "goodbye"],
        }
    ]
}


print(clean_nones(a))

results in this:

{
    'b': 'notNone',
    'c': ['hello', 'goodbye'],
    'd': [
        {
            'a': 'notNone',
            'c': ['hello', 'goodbye']
        },
        {
            'a': 'notNone',
            'c': ['hello', 'goodbye']
        }
    ]
}

Answer 7

def del_none(d):
    """
    Delete keys with the value ``None`` in a dictionary, recursively.

    This alters the input so you may wish to ``copy`` the dict first.
    """
    # For Python 3, write `list(d.items())`; `d.items()` won’t work
    # For Python 2, write `d.items()`; `d.iteritems()` won’t work
    for key, value in list(d.items()):
        if value is None:
            del d[key]
        elif isinstance(value, dict):
            del_none(value)
    return d  # For convenience

Sample usage:

>>> mydict = {'dict1': {'key1': 'value1', 'key2': None}}
>>> print(del_none(mydict.copy()))
{'dict1': {'key1': 'value1'}}

Then you can feed that to json.

Answer 8

>>> def cleandict(d):
...     if not isinstance(d, dict):
...         return d
...     return dict((k,cleandict(v)) for k,v in d.iteritems() if v is not None)
... 
>>> mydict = dict(dict1=dict(key1='value1', key2=None))
>>> print cleandict(mydict)
{'dict1': {'key1': 'value1'}}
>>> 

I don't like using del in general, changing the existing dictionary can have subtle effects depending on how they are created. Creating new dictionaries with None removed prevents all side effect.

Answer 9

def excludeNone(d):
    for k in list(d):
        if k in d:
            if type(d[k]) == dict:
                excludeNone(d[k])
            if not d[k]:
                del d[k]