flask marshmallow tables serialization failing

Question

I have my model that represent 4 Tables: User ,Test,Area,Issue One User has multiple Test.Each Test has multiples Areas, and each Area has multiple Issues I want to refactor the code(that is working ok) to use marshmallow for serialize my SLQ-Alchemy objects

The fist method using marshmallow works ok. However Im having issues when trying to return one test , with all its areas and issues.

So here is the details:

This is my model.py file

db = SQLAlchemy()
ma = Marshmallow()


class User(db.Model):

    __tablename__ = 'user'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(250), nullable=False)
    email = db.Column(db.String(250), nullable=False)

class Test(db.Model):

    __tablename__ = 'test'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, ForeignKey('user.id'))
    user = relationship(User, backref=backref('tests'))

class Area(db.Model):

    __tablename__ = 'area'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(30), nullable=False)
    test_id = db.Column(db.Integer, ForeignKey('test.id'))
    test = relationship(Test,
                        backref=backref('areas', cascade='all, delete-orphan')
                        )
    user_id = db.Column(db.Integer, ForeignKey('user.id'))
    user = relationship(User, backref=backref('areas'))

class Issue(db.Model):

    __tablename__ = 'issue'
    name = db.Column(db.String(80), nullable=False)
    id = db.Column(db.Integer, primary_key=True)
    area_id = db.Column(db.Integer, ForeignKey('area.id'))
    area = relationship(Area,
                        backref=backref('issues', cascade='all, delete-orphan')
                        )
    user_id = db.Column(db.Integer, ForeignKey('user.id'))
    user = relationship(User, backref=backref('issues'))


class UserSchema(ma.ModelSchema):
    class Meta:
        model = User
    # A User has a list of tests
    test = ma.Nested('TestSchema', many=True)


class TestSchema(ma.ModelSchema):
    class Meta:
        model = Test
    # Each test belongs to one user
    user = ma.Nested('UserSchema')
    # Each test has a list of areas
    area = ma.Nested('AreaSchema', many=True)


class AreaSchema(ma.ModelSchema):
    class Meta:
        model = Area
    # Each Area belongs to one test
    test = ma.Nested('TestSchema')
    # Each Area has a list of issues
    issue = ma.Nested('IssueSchema', many=True)


class IssueSchema(ma.ModelSchema):
    class Meta:
        model = Issue
    # Each issue belongs to one area
    area = ma.Nested('AreaSchema')

So now here is the Resource that is working perfectly using marshmallow. This returns all the tests , WITHOUT its areas and WITHOUT its Issues This is working OK:

# This works perfectly .
# It returns all tests for the user without its Areas and without Issues
# See line 26 the use of tests = tests_schema.dump(results).data

tests_schema = TestSchema(many=True)

class Tests(Resource):
    """ This method return the list of tests for the user
    We expect a valid JWT token from the user   that was already
    validated thorugh the decorator we created: token_required"""
    @token_required
    def get(self, *args, **kwargs):
        jwt_data = kwargs['jwt_data']
        if jwt_data:
            # The JWT payload contains the "user"
            user_name = jwt_data["user"]
            logger.info("User from JWT payload data is  %s" % user_name)
            userId = modelUser.getUserIDFromName(user_name)
            user = modelUser.getUserInfo(userId)
            results = modelTest.getAllTestsForUser(userId)
            logger.info(results)
            tests = tests_schema.dump(results).data
            logger.info(tests)
            if tests:
                return jsonify(tests)
            else:
                response = make_response(json.dumps(
                                        'You do not have any test'), 204)
                response.headers['Content-Type'] = 'application/json'
                return response

Here is where I have the problem Im getting an empty dictionary in :

result = test_schema.dump(testWithAreasAndIssues).data

# This returns just one test with ALL its Areas and ALL ISSUES
# The value returned from the DB is correct.I'm just refactoring to use Marshmallow
# line  19 returns empty

class Test(Resource):
    """ GET DELETE AND PUT(modifies) a Test /api/test/<int:test_id>"""
    @token_required
    def get(self, test_id, *args, **kwargs):
        logger.info("GET for /api/test/test_id= %s" % test_id)
        jwt_data = kwargs['jwt_data']
        test = getTestForJWT(jwt_data, test_id)
        logger.info('test.id= %s for jwt=%s is %s' % (test_id, jwt_data, test))
        logger.info('test= %s' % test)
        logger.info('Calling getTestWithAreasAndIssues')
        testWithAreasAndIssues = modelTest.getTestWithAreasAndIssues(
                                        test.id)
        logger.info('FullTest for testid =%s is %s' % (
                                    test.id, testWithAreasAndIssues))
        result = test_schema.dump(testWithAreasAndIssues).data
        logger.info(jsonify(result))

Why Im getting an empty dictionary in
result = test_schema.dump(testWithAreasAndIssues).data
?

Here is the function in the model that get the test with all its area and issue.

def getTestWithAreasAndIssues(id):
        """ This method will return a table containing a test
        with all its areas and each areas with all its issues.
        The result are unserialized object in a table with 3 columns
        So we need to later serialize them and convert them
        to a herarquick view using a python dictionary"""
        test = (db.session.query(Test, Area, Issue)
                .join(Area)
                .join(Issue)
                .options(
                    joinedload(Test.areas)
                    .joinedload(Area.issues)
                )
                .filter(Test.id == id)
                .filter(Test.id == Area.test_id)
                .filter(Area.id == Issue.area_id)
                ).all()
         return test

This is the output of this function :

[(<Test 4>, <Area 28>, <Issue 17>), 
 (<Test 4>, <Area 29>, <Issue 18>), 
 (<Test 4>, <Area 36>, <Issue 19>), 
 (<Test 4>, <Area 36>, <Issue 20>), 
 (<Test 4>, <Area 36>, <Issue 21>)]

Before using marshmallow I created a function that took this SQLAlchemy table and turned into a python object .

Answer 1

I got what I wanted with:

test_schema = TestSchema(exclude=('user',))
areas_schema = AreaSchema(many=True, exclude=('test', 'user',))
issues_schema = IssueSchema(many=True, exclude=('test', 'user',))

and later:

test, error = test_schema.dump(test_result)
areas, error = areas_schema.dump(test_result.areas)
issues, error = issues_schema.dump(test_result.issues)
return jsonify({'test': test, 'areas': areas, 'issues': issues})

If someone get to know why

test_detailed_schema = Test_DetailedSchema()

with

class Test_DetailedSchema(ma.ModelSchema):
    test = ma.Nested('TesSchema')
    areas = ma.Nested('AreaSchema', many=True, exclude=('test', 'user',))
    issues = ma.Nested('IssueSchema', many=True, exclude=('test', 'user',))

Do not return the same result , please let the response here

Answer 2

I manage to create something similar to what I want by doing some changes in the Model ( I added a backref and a relation from Issue to Test in the model). So here is my Schemas now

class UserSchema(ma.ModelSchema):

    class Meta:
        model = User

class TestSchema(ma.ModelSchema):

    class Meta:
        model = Test

class AreaSchema(ma.ModelSchema):

    class Meta:
        model = Area

class IssueSchema(ma.ModelSchema):

    class Meta:
        model = Issue

class Test_DetailedSchema(ma.ModelSchema):
    test = ma.Nested('self')
    areas = ma.Nested('AreaSchema', many=True, exclude=('test', 'user',))
    issues = ma.Nested('IssueSchema', many=True,
                       include=('name', 'id', 'reference_number', 'status',))

So now in my views I do

from models.model import TestSchema
from models.model import IssueSchema
from models.model import AreaSchema
from models.model import Test_DetailedSchema

# Schemas
test_detailed_schema = Test_DetailedSchema()
test_schema = TestSchema(exclude=('user',))
areas_schema = AreaSchema(many=True, exclude=('test', 'user',))
issues_schema = IssueSchema(many=True)

And in the route I do something like this :

class Test(Resource):
    """ GET DELETE AND PUT(modifies) a Test /api/test/<int:test_id>"""
    @token_required
    def get(self, test_id, *args, **kwargs):

        test_result = modelTest.getTest(test_id)
        test_details, error = test_detailed_schema.dump(test_result)
        pprint.pprint({'test_details': test_details})

This is the output I got:

{'test': {'areas': [
                            {'id': 10, 'issues': [7, 8], 'name': 'Name1'},
                            {'id': 11, 'issues': [9], 'name': 'NameX'},
                            {'id': 12, 'issues': [], 'name': 'Name2'},
                            {'id': 13,'issues': [],'name': 'Name3'},
                            {'id': 14, 'issues': [], 'name': 'Name4'},
                            {'id': 15,'issues': [],'name': 'Name5'},
                            {'id': 16, 'issues': [], 'name': 'Name6'},
                            {'id': 17, 'issues': [], 'name': 'Name7'},
                            {'id': 18,'issues': [10, 11],'name': 'Name8'}],
     'issues': [{
                              'area': 10,
                              'id': 7,
                              'name': 'This is the issueX',
                              'open_date': None,
                              'reference_number': '701',
                              'status': 'N',
                              'test': 2,
                              'user': 1},
                             {'area': 10,
                              'id': 8,
                              'name': 'This is the issueY',
                              'open_date': None,
                              'reference_number': '702',
                              'status': 'N',
                              'test': 2,
                              'user': 1},
                             {'area': 11,
                              'id': 9,
                              'name': 'This is the issueZ',
                              'open_date': None,
                              'reference_number': '703',
                              'status': 'N',
                              'test': 2,
                              'user': 1},
                             {'area': 18,
                              'id': 10,
                              'name': 'This is the issueZZ',
                              'open_date': None,
                              'reference_number': '786',
                              'status': 'N',
                              'test': 2,
                              'user': 1},
                             {'area': 18,
                              'id': 11,
                              'name': 'This is the issueXXC',
                              'open_date': None,
                              'reference_number': '787',
                              'status': 'N',
                              'test': 2,
                              'user': 1}]}}

So wha should I do to expand the Issues inside the areas and avoid:

'id': 10, 'issues': [7, 8], 'name': 'Name1'}

and have instead

{'test': {'areas': [
                                { 'id': 10,
                                 'name': 'Name1' 
                                 'issues':[
                                  {'area': 10,
                                  'id': 7,
                                  'name': 'This is the issueX',
                                  'open_date': None,
                                  'reference_number': '701',
                                  'status': 'N',
                                  'test': 2,
                                  'user': 1},
                                 {'area': 10,
                                  'id': 8,
                                  'name': 'This is the issueY',
                                  'open_date': None,
                                  'reference_number': '702',
                                  'status': 'N',
                                  'test': 2,
                                  'user': 1}
                                 ] 

Why issues are not expanding inside areas ?

Answer 3

You should use a different schema to be used by getTestWithAreasAndIssues(). Starting by having a TestSchema that correctly corresponds to your Test model:

class TestSchema(ma.ModelSchema):
    class Meta:
        model = Test

I would also recommend that you review your models, your User model does not contain a relationship with Test, Area or Issue. Have a look here to correctly define relationships with SQLAlchemy.

Then you can have a Schema for the results returned by getTestWithAreasAndIssues():

class TestSchema(ma.ModelSchema):
    test = ma.Nested('TestSchema')
    user = ma.Nested('UserSchema')
    area = ma.Nested('AreaSchema')