Mongoose Index not being Created before Query is Run

Question

When running my tests I'm getting an error stating I don't have a text index setup on the title field yet when running my application text searches work fine with this same model and don't throw the error.

text index required for $text query (no such collection 'test-db.torrents')

import mongoose from 'mongoose';
import Category from './category';

const Schema = mongoose.Schema;

const Torrent = new Schema({
    title: {
        type: String
    },
    category: {
        type: mongoose.Schema.Types.ObjectId,
        ref: 'Category',
        required: true,
        index: true
    },
    size: Number,
    details: [
        {
            type: String
        }
    ],
    swarm: {
        seeders: Number,
        leechers: Number
    },
    lastmod: {
        type: Date,
        default: Date.now()
    },
    imported: {
        type: Date,
        default: Date.now()
    },
    infoHash: {
        type: String,
        unique: true,
        index: true
    }
});

Torrent.index({
    title: 'text'
}, {
    background: false
});

export default mongoose.model('Torrent', Torrent);

I'm using ava for testing and here is my test case.

import mongoose from 'mongoose';
import request from 'supertest';
import test from 'ava';
import {makeApp} from '../helpers';

test.before(() => {
    mongoose.Promise = Promise;
    mongoose.connect('mongodb://localhost:27017/astro-test-db');
});

test.after.always(() => {
    mongoose.connection.db.dropDatabase(() => {
        mongoose.disconnect();
    });
});

// Should return [] since HL3 doesn't exist.
test('should return no search results', async t => {
    const app = makeApp();
    const res = await request(app).get(`/api/search?q=HL3`);

    t.is(res.status, 200);
    t.is(res.body.error, {});
    t.is(res.body.torrents.length, 0);
});

Here's the full ouput from ava, as you can see the index for title doesn't get created with 'text' or background: false.

➜  astro git:(develop) ✗ yarn ava test/routes/search.js -- --verbose
yarn ava v0.24.6
$ "/Users/xo/code/astro/node_modules/.bin/ava" test/routes/search.js --verbose

Mongoose: categories.ensureIndex({ title: 1 }, { unique: true, background: true })
Mongoose: torrents.ensureIndex({ category: 1 }, { background: true })
Mongoose: torrents.count({}, {})
Mongoose: categories.ensureIndex({ slug: 1 }, { unique: true, background: true })
Mongoose: torrents.find({ '$text': { '$search': 'x' } }, { limit: 100, sort: { score: { '$meta': 'textScore' }, 'swarm.seeders': -1 }, fields: { score: { '$meta': 'textScore' } } })
  ✖ should return no search results 

  1 test failed [13:59:07]

  should return no search results
  /Users/xo/code/astro/test/routes/search.js:24

   23:     t.is(res.status, 200);            
   24:     t.is(res.body.error, {});         
   25:     t.is(res.body.torrents.length, 0);

  Difference:

    - Object {
    -   code: 27,
    -   codeName: "IndexNotFound",
    -   errmsg: "text index required for $text query (no such collection \'astro-test-db.torrents\')",
    -   message: "text index required for $text query (no such collection \'astro-test-db.torrents\')",
    -   name: "MongoError",
    -   ok: 0,
    - }
    + Object {}

  _callee$ (test/routes/search.js:24:7)
  tryCatch (node_modules/regenerator-runtime/runtime.js:65:40)
  Generator.invoke [as _invoke] (node_modules/regenerator-runtime/runtime.js:303:22)
  Generator.prototype.(anonymous function) [as next] (node_modules/regenerator-runtime/runtime.js:117:21)
  step (test/routes/search.js:19:191)

error Command failed with exit code 1.

Answer 1

You should make sure the index is created in the "foreground" since "background" creation is default.

Torrent.index({
    title: 'text'
},{ "background": false });

At least for your tests, otherwise there is a possibility that the query can run before the index is created. Setting { background: false } makes sure the index is there before other operations can run. This is opposite to the default behavior of MongoDB, so it needs to be an explicit setting.

In production environments it is usually best to deploy indexes by other means anyway. Also "foreground" creation results in a smaller index size but of course "blocks" IO, but it's still generally better to do this at least once in production.

Citation from documentation

By default, MongoDB builds indexes in the foreground, which prevents all read and write operations to the database while the index builds. Also, no operation that requires a read or write lock on all databases (e.g. listDatabases) can occur during a foreground index build.

Which means no reads or writes can occur when this happens. So data cannot be inserted and queries cannot be run whilst indexes are being created when in "foreground" creation mode.

As for size, a little further down the same page from the citation:

Background index builds take longer to complete and result in an index that is initially larger, or less compact, than an index built in the foreground. Over time, the compactness of indexes built in the background will approach foreground-built indexes.

So you can create indexes in the background that will "over time" come down to a more compact size in a production environment. But for testing and development purposes, your default should really always be to create in "foreground" in order not to be caught out with timing issues.

---

As a minimal test case:

var mongoose = require('mongoose'),
    Schema = mongoose.Schema;

mongoose.set('debug', true);

var testSchema = new Schema({
  title: { type: String }
});

testSchema.index({
  title: 'text'
},{ background: false });

var Test = mongoose.model('Test', testSchema);

mongoose.connect('mongodb://localhost/texttest');

Test.create({ title: 'something here' },function(err,doc) {

  Test.find({ "$text": { "$search": "something" } },function(err,doc) {
    if (err) throw err;
   console.log(doc);

   Test.collection.drop(function(err) {
    if (err) throw err;
    mongoose.disconnect();
   });
  });
});

As an alternate approach, turn off mongooses autoindex feature automatically and set manually, then call creation manually via .ensureIndexes():

var async = require('async'),
    mongoose = require('mongoose'),
    Schema = mongoose.Schema;

mongoose.set('debug', true);

var testSchema = new Schema({
  title: { type: String }
},{ autoIndex: false });

testSchema.index({
  title: 'text'
},{ background: false });

var Test = mongoose.model('Test', testSchema);

mongoose.connect('mongodb://localhost/texttest');

// Manually set indexing to on
Test.schema.options.autoIndex = true;
//console.log(Test.schema.options);

Test.ensureIndexes(function(err) {
  if (err) throw err;

  Test.create({ title: 'something here' },function(err,doc) {

    Test.find({ "$text": { "$search": "something" } },function(err,doc) {
      if (err) throw err;
      console.log(doc);

      Test.collection.drop(function(err) {
        if (err) throw err;
        mongoose.disconnect();
      });
    });
  });
});