Lookup with array of Objects

Question

I am trying to populate two object arrays of my document which reference to another collection. However it always appears as an empty array in the results which surprises me. What am I doing wrong here? Is there a better way to 'populate' uploaded_files and file_history?

That's my aggregate statement:

    Project.aggregate(
        { $match: {"project_id": projectId}},
        { $addFields: {
            uploaded_files: {
                $filter: {
                    input: '$uploaded_files',
                    as: 'uploadedFile',
                    cond: {$eq: ['$$uploadedFile.upload_id', uploadId]}
                }
            },
            file_history: {
                $filter: {
                    input: '$file_history',
                    as: 'revision',
                    cond: {$eq: ['$$revision.upload_id', uploadId]}
                }
            },
        }},
        { $lookup: {
            from: 'users',
            localField: 'owner',
            foreignField: '_id',
            as: 'owner'
        }},
        { $lookup: {
            from: 'files',
            localField: 'uploaded_files.file',
            foreignField: '_id',
            as: 'test'
        }},
        { $lookup: {
            from: 'files',
            localField: 'file_history.file',
            foreignField: '_id',
            as: 'test2'
        }},
        { $unwind: '$owner' },
        { $limit: 1 }
    ).then(projects => {
        if (projects.length == 0)
            return res.status(404).send({ error: "Couldn't find a project with this id" })

        let project = projects[0]
})

And my document looks like this:

{
    "_id" : ObjectId("5935a41f12f3fac949a5f925"),
    "project_id" : 13,
    "updated_at" : ISODate("2017-07-09T19:41:51.396Z"),
    "created_at" : ISODate("2017-06-05T18:34:07.150Z"),
    "owner" : ObjectId("591eea4439e1ce33b47e73c3"),
    "name" : "Demo project",
    "uploaded_files" : [ 
        {
            "display_name" : "001.jpg",
            "file" : ObjectId("596286ff7d3a594ed4797848"),
            "upload_id" : ObjectId("596286ff7d3a594ed4797849"),
            "created_at" : ISODate("2017-07-09T19:41:51.000Z")
        }
    ],
    "file_history" : [ 
        {
            "display_name" : "001.jpg",
            "file" : ObjectId("596286ff7d3a594ed4797848"),
            "upload_id" : ObjectId("596286ff7d3a594ed4797849"),
            "created_at" : ISODate("2017-07-09T19:41:51.000Z")
        }
    ]
}

Answer 1

You basically need to $unwind the array first. MongoDB cannot yet work with the "inner" property of an object within an array as a source for $lookup.

Also for efficiency we really should use $concatArrays first to "join" the array source, and then only do one $lookup operation:

Project.aggregate([
  { "$match": { "project_id": projectId} },
  { "$project": {
    "project_id": 1,
    "updated_at": 1,
    "created_at": 1,
    "owner": 1,
    "name": 1,
    "combined": {
      "$concatArrays": [
        { "$map": {
          "input": {
            "$filter": {
              "input": "$uploaded_files",
              "as": "uf",
              "cond": { "$eq": ["$$uf.upload_id", uploadId ] }
            }
          },
          "as": "uf",
          "in": {
            "$arrayToObject": {
              "$concatArrays": [
                { "$objectToArray": "$$uf" },
                [{ "k": "type", "v": "uploaded_files" }]
              ]
            }
          }
        }},
        { "$map": {
          "input": {
            "$filter": {
              "input": "$file_history",
              "as": "fh",
              "cond": { "$eq": ["$$fh.upload_id", uploadId ] }
            }
          },
          "as": "fh",
          "in": {
            "$arrayToObject": {
              "$concatArrays": [
                { "$objectToArray": "$$fh" },
                [{ "k": "type", "v": "file_history" }]
              ]
            }
          }
        }}
      ]
    }
  }},
  { "$unwind": "$combined" },
  { "$lookup": {
    "from": "files",
    "localField": "combined.file",
    "foreignField": "_id",
    "as": "combined.file"
  }},
  { "$unwind": "$combined.file" },
  { "$lookup": {
    "from": "users",
    "localField": "owner",
    "foreignField": "_id",
    "as": "owner"
  }},
  { "$unwind": "$owner" },
  { "$group": {
    "_id": "$_id",
    "project_id": { "$first": "$project_id" },
    "updated_at": { "$first": "$updated_at" },
    "created_at": { "$first": "$created_at" },
    "owner": { "$first": "$owner" },
    "name": { "$first": "$name" },
    "combined": { "$push": "$combined" }
  }},
  { "$project": {
    "project_id": 1,
    "updated_at": 1,
    "created_at": 1,
    "owner": 1,
    "name": 1,
    "uploaded_files": {
      "$filter": {
        "input": "$combined",
        "as": "cf",
        "cond": { "$eq": [ "$$cf.type", "uploaded_files" ] }
      }    
    },
    "file_history": {
      "$filter": {
        "input": "$combined",
        "as": "cf",
        "cond": { "$eq": [ "$$cf.type", "file_history" ] }
      }    
    }
  }}
])

In a nutshell

1. Bring the two arrays together in on source and tag them, then $unwind first

2. Do the $lookup on the combined detail and $unwindthat

3. Do the $lookup on the other foreign source and $unwind that

4. $group the document back together with a single array.

5. $filter by the "tag names" or "type" field we added to "separate" the arrays.

You can follow the same sort of process just be simply using $unwind on each array, then doing the "join" and grouping back together. But really that needs a lot more steps, than simply "combining" in the first place.

Also note that $lookup followed by $unwind is actually treated as one pipeline stage when processed on the server. See Aggregation Pipeline Optimization for details