Django REST Framework getting object ID back from hyperlinked identity field

Question

I am having difficulty getting the URL (or any sort of ID) back from the validated_data when using a HyperlinkedIdentityField. I need the URL because I am doing writable nested representations and I want to know if the nested representation refers to an existing object (url specified) or a new object (url not specified). I have read the example in the docs about customizing ListSerializer update but that seems to rely on the client sending an id that does not exist, which does not work for me as I want to use URLs instead. I have checked validated_data for a top-level (not nested) object with a HyperlinkedIdentityField and the ID is not exposed there either.

EDIT: I am now aware this is because HyperlinkedIdentityField sets the read_only attribute and read-only fields are skipped when generating validated data. Tips welcome on how to get around this.

I have two models in this phonebook app: Entry and Phone. An Entry has multiple Phones. Just in case this is relevant, the model code is as follows:

from django.db import models


class Entry(models.Model):
    name = models.CharField(max_length=200)


class Phone(models.Model):
    type = models.CharField(max_length=20)
    number = models.CharField(max_length=50)
    parent = models.ForeignKey(Entry, related_name='phones')

My serializer definitions follow:

class PhoneListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data): 
        print repr(validated_data)
        url_of = lambda p : self.child.to_representation(p)['url']
        existing_instances = { url_of(p): p for p in instance }
        existing_submitted_instances = { item['url']: item for item in validated_data if 'url' in item }
        new_submitted_instances = [ item for item in validated_data if 'url' not in item ]
        urls_to_delete = existing_instances.viewkeys() - existing_submitted_instances.viewkeys()                                                       
        objects_to_delete = [existing_instances[u] for u in urls_to_delete if u in existing_instances]                                                 
        objects_to_update = [(existing_instances[u], p) for u, p in six.iteritems(existing_submitted_instances) if u in existing_instances]

        result = []
        for o in objects_to_delete:
            o.delete()
        for existing, data in objects_to_update:                          
            result.append(self.child.update(existing, data))
        for data in new_submitted_instances:                              
            data['parent'] = self.root.instance
            result.append(self.child.create(data))
        return result

class PhoneSerializer(serializers.HyperlinkedModelSerializer):
    url = serializers.HyperlinkedIdentityField(
        view_name='phone', lookup_field='id')
    class Meta:
        model = Phone
        list_serializer_class = PhoneListSerializer
        fields = ('type', 'number', 'url')

class EntrySerializer(serializers.HyperlinkedModelSerializer):
    url = serializers.HyperlinkedIdentityField(
        view_name='entry', lookup_field='id')
    phones = PhoneSerializer(many=True, required=False)

    class Meta:
        model = Entry
        fields = ('url', 'name', 'phones')

    def update(self, instance, validated_data):
        print repr(validated_data)
        # pop this first so super does not complain about writable nested
        # serializers. we will update phones ourselves.
        phone_data = validated_data.pop('phones', [])
        phones_field = self.fields['phones']

        instance = super(EntrySerializer, self).update(instance, validated_data)

        phones_field.update(instance.phones.all(), phone_data)

        return instance

    def create(self, validated_data):
        phone_data = validated_data.pop('phones', [])
        new_entry = Entry.objects.create(**validated_data)
        # TODO atomically do this
        for phone_validated_data in phone_data:
            Phone.objects.create(parent=new_entry, **phone_validated_data)
        return new_entry

Currently getting data out works, but resubmitting the data (even with the URL) results in all phones of an entry being deleted and then recreated with new IDs.

EDIT: attempting to use a HyperlinkedRelatedField (and a ModelSerializer instead of a HyperlinkedModelSerializer) gets further (the field succeeds in pulling an object out), but still fails when the serializer calls fields.set_value

Code:

class PhoneSerializer(serializers.ModelSerializer):
    url = serializers.HyperlinkedRelatedField(
        required=False, view_name='phone', lookup_field='id',
        queryset=Phone.objects, source='*')

This fails since it eventually calls rest_framework.fields.set_value(validated_data, [], <Phone object>) and it will call validated_data.update(<Phone object>) which fails with 'Phone object is not iterable'.

Traceback (most recent call last):
  [...snip...]
  File "/app/phonebook_be/views.py", line 41, in entry
    if serializer.is_valid(raise_exception=True):
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 191, in is_valid
    self._validated_data = self.run_validation(self.initial_data)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 371, in run_validation
    value = self.to_internal_value(data)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 404, in to_internal_value
    validated_value = field.run_validation(primitive_value)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 522, in run_validation
    value = self.to_internal_value(data)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 552, in to_internal_value
    validated = self.child.run_validation(item)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 371, in run_validation
    value = self.to_internal_value(data)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 414, in to_internal_value
    set_value(ret, field.source_attrs, validated_value)
  File "/venv/lib/python2.7/site-packages/rest_framework/fields.py", line 96, in set_value
    dictionary.update(value)
  File "/venv/lib/python2.7/_abcoll.py", line 568, in update
    for key, value in other:
TypeError: 'Phone' object is not iterable

Other attempts:

source='id': results in error when rendering

Traceback (most recent call last):
  [...snip...]
  File "/app/phonebook_be/views.py", line 18, in entries
    json_data = JSONRenderer().render(serializer.data)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 470, in data
    ret = super(Serializer, self).data
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 217, in data
    self._data = self.to_representation(self.instance)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 439, in to_representation
    ret[field.field_name] = field.to_representation(attribute)
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 572, in to_representation
    self.child.to_representation(item) for item in iterable
  File "/venv/lib/python2.7/site-packages/rest_framework/serializers.py", line 439, in to_representation
    ret[field.field_name] = field.to_representation(attribute)
  File "/venv/lib/python2.7/site-packages/rest_framework/relations.py", line 264, in to_representation
    return self.get_url(value, self.view_name, request, format)
  File "/venv/lib/python2.7/site-packages/rest_framework/relations.py", line 202, in get_url
    lookup_value = getattr(obj, self.lookup_field)
AttributeError: 'int' object has no attribute 'id'

Answer 1

You are currently manually comparing URLs. That works however there are some issues:

url_of = lambda p : self.child.to_representation(p)['url']
existing_instances = { url_of(p): p for p in instance }

the second line is expensive since the instance is a queryset so as soon as you will have lots of data, that can become a memory hog since you will be creating a dict for all the instances matching queryset.

Instead of doing that, DRF natively supports parsing URLs in order to get the identity object:

class PhoneSerializer(serializers.ModelSerializer):
     url = serializers.HyperlinkedRelatedField(
         required=False,
         view_name='phone',
         lookup_field='id',
     )

Notice the use of HyperlinkedRelatedField. Unlike HyperlinkedIdentityField (which subclasses HyperlinkedRelatedField), HyperlinkedRelatedField is not read-only. It is capable of parsing the URL and getting the model out of the URL itself by reversing the URL and then using the kwargs in the url to lookup the object in db. As a result, you can rely on that behavior to lookup existing objects in db. Notice that the field altogether is optional which allows the client of the API to omit it in which case URL will not be parsed hence a new object can be created.

class PhoneListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data): 
        to_update = filter(lambda i: i.get('url'), validated_data)
        to_create = filter(lambda i: not i.get('url'), validated_data)
        data = []
        for i in to_update:
            data.append(self.child.update(i['url'], i))
        for i in to_create:
            data.append(self.child.create(i))
        return data

Hopefully this helps. Note that I did this from memory so there might be some things to watch out for but this concept should generally work.

Finally there are some docs about the HyperlinkedRelatedField - http://www.django-rest-framework.org/api-guide/relations/#hyperlinkedrelatedfield

Answer 2

So to work around this I created a write-only field (inspired by HiddenField) on the PhoneSerializer that takes its value from the submitted 'url' value:

class WriteOnlySynonymField(serializers.Field):

    def __init__(self, **kwargs):
        kwargs['default'] = serializers.empty
        kwargs['required'] = False
        kwargs['write_only'] = True
        self.synonym_for = kwargs.pop('synonym_for')
        super(WriteOnlySynonymField, self).__init__(**kwargs)

    def get_value(self, dictionary):
        return dictionary.get(self.synonym_for, serializers.empty)

    def to_internal_value(self, data):
        return data


class PhoneSerializer(serializers.ModelSerializer):
    url = MultiKeyHyperlinkedIdentityField(
        view_name='phone',
        lookup_kwarg_to_field={'id': 'id', 'entry_id': 'parent_id'})
    submitted_url = WriteOnlySynonymField(synonym_for='url')

    class Meta:
        model = Phone
        list_serializer_class = PhoneListSerializer
        fields = ('type', 'number', 'url', 'submitted_url')

And the relevant bits of the list serializer update changes slightly to use the new field name:

existing_submitted_instances = {item['submitted_url']: item
                                for item in validated_data
                                if 'submitted_url' in item}
new_submitted_instances = [item for item in validated_data
                           if 'submitted_url' not in item]