How to get data from all objects using tastypie API?

Question

My goal is to get all data from tastypie API. This API has more than 1.000.000 objects inside! I have created another Django project and a custom "product_import.py" command which should get all data from this API.

What should I do to continue getting data from API when I reach the limit? I want to get data from API ['meta']['next'] dictionary to continue collecting objects in another project.

Below is an example I created for now and got stuck.

api.py

# coding: utf-8

from tastypie.resources import ModelResource
from tastypie import fields
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import DjangoAuthorization

from .models import Product


class UserguideResource(ModelResource):

    class Meta:
        resource_name = 'product'
        allowed_methods = ['get']
        fields = ['id', 'name', 'date_added', 'author']
        queryset = Product.objects.all()
        authentication = ApiKeyAuthentication()
        authorization = DjangoAuthorization()

This file is created in different Django project which uses tastypie API from project above. I want to get all data from this API.

product_import.py

from django.core.management.base import BaseCommand
import requests


class Command(BaseCommand):
    """ Import Products from API """

    def __init__(self):
        self.API_KEY = 'secret-key'
        self.API_USERNAME = 'keNzi'
        self.product_api_url = 'http://localhost:8000/api/v1/product?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.limit = 1
        self.offset = 0

    def get_latest_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response

    def get_next_url_from_product_list(self):
        return self.get_latest_product_from_api()['meta']['next']

    def get_next_product_list(self):
        self.next = self.get_next_url_from_product_list()
        response = requests.get(
            "{0}{1}".format(
                self.product_api_url,
                self.next,
            )
        ).json()
        return response

    def handle(self, *args, **options):
        print(self.get_latest_product_from_api())
        print(self.get_next_product_list())

print() displays products from API but I want to continue and gather all data from it.

Answer 1

Here is a code which works fine ;)

import requests
from django.core.management.base import BaseCommand
from product.models import Product


class Command(BaseCommand):
    """ Import Products from API """

    def __init__(self):
        self.API_URL = 'http://localhost:8000/api/v1'
        self.API_USERNAME = 'keNzi'
        self.API_KEY = 'keNzi-secret-2017'   # created in /admin/tastypie/apikey/
        self.API_AUTH = '?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.product_api_url = '{0}/product/{1}'.format(
            self.API_URL, self.API_AUTH)
        self.limit = 1
        self.offset = 0

    def get_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response

    def get_next_url_from_product_api(self):
        return self.get_product_from_api()['meta']['next']

    def handle(self, *args, **options):
        next_url = True
        while next_url:
            for obj in self.get_product_from_api()['objects']:
                print(obj['id'], obj['file'])
                product = Product.objects.create(
                    name=obj['name'],
                    description=obj['text'],
                    created=obj['date_added'],
                )
                product.save()
                print('Product saved! ID of the new objects: ', product.id)

            self.offset += 1
            next_url = self.get_next_url_from_product_api()