CODEWITHSUNDEEP
lucene
user605331
user605331

Reputation: 3788

How to use prefix queries on fields in Lucene?

I am trying to use a PrefixQuery in Lucene for the purpose of autocomplete. I've made a simple test of what I thought should work, but it doesn't. I am indexing some simple strings and using the KeywordAnalyzer to make sure they are not tokenized, but my searches still do not match anything. How should I index and search a field to get a prefix match?

Here's the unit test I made to test with. Everything passes except the autocomplete and singleTerm methods.

package com.sample.index;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.HashMap;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;

public class TestIndexStuff {
    public static final String FIELD_AUTOCOMPLETE = "autocomplete";
    public static final String FIELD_NORMAL = "normal";
    private IndexSearcher searcher;
    private PerFieldAnalyzerWrapper analyzer;

    @Before
    public void init() throws IOException {
        RAMDirectory idx = new RAMDirectory();

        HashMap<String, Analyzer> fieldAnalyzers = new HashMap<String, Analyzer>();
        fieldAnalyzers.put(FIELD_AUTOCOMPLETE, new KeywordAnalyzer());
        analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(Version.LUCENE_35), fieldAnalyzers);
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35, analyzer);

        IndexWriter writer = new IndexWriter(idx, config);
        addDocs(writer);
        writer.close();

        searcher = new IndexSearcher(IndexReader.open(idx));
    }

    private void addDocs(IndexWriter writer) throws IOException {
        for (String text : new String[]{"Fred Rogers", "Toni Reed Preckwinkle", "Randy Savage", "Kathryn Janeway", "Madonna", "Fred Savage"}) {
            Document doc = new Document();
            doc.add(new Field(FIELD_NORMAL, text, Field.Store.YES, Field.Index.ANALYZED));
            doc.add(new Field(FIELD_AUTOCOMPLETE, text, Field.Store.YES, Field.Index.NOT_ANALYZED));
            writer.addDocument(doc);
        }

    }

    @Test
    public void prefixParser() throws ParseException {
        Query prefixQuery = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fre*");
        assertTrue(prefixQuery instanceof PrefixQuery);

        Query normalQuery = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fred");
        assertFalse(normalQuery instanceof PrefixQuery);
    }

    @Test
    public void normal() throws ParseException, IOException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_NORMAL, analyzer).parse("Fred");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(2, topDocs.totalHits);
    }

    @Test
    public void autocomplete() throws IOException, ParseException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fre*");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(2, topDocs.totalHits);
    }

    @Test
    public void singleTerm() throws ParseException, IOException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Mado*");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(1, topDocs.totalHits);
    }
}

edit: adding revised code for those who read this later to show full test after changing thanks to @jpountz. Rather than leave things as mixed case though, I chose to index them as lower case. I also added a unit test to make sure that a term in the middle would not be matched, since this should only match things that start with the search term.

package com.sample.index;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.HashMap;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;

public class TestIndexStuff {
    public static final String FIELD_AUTOCOMPLETE = "autocomplete";
    public static final String FIELD_NORMAL = "normal";
    private IndexSearcher searcher;
    private PerFieldAnalyzerWrapper analyzer;

    @Before
    public void init() throws IOException {
        RAMDirectory idx = new RAMDirectory();

        HashMap<String, Analyzer> fieldAnalyzers = new HashMap<String, Analyzer>();
        fieldAnalyzers.put(FIELD_AUTOCOMPLETE, new KeywordAnalyzer());
        analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(Version.LUCENE_35), fieldAnalyzers);
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35, analyzer);

        IndexWriter writer = new IndexWriter(idx, config);
        addDocs(writer);
        writer.close();

        searcher = new IndexSearcher(IndexReader.open(idx));
    }

    private void addDocs(IndexWriter writer) throws IOException {
        for (String text : new String[]{"Fred Rogers", "Toni Reed Preckwinkle", "Randy Savage", "Kathryn Janeway", "Madonna", "Fred Savage"}) {
            Document doc = new Document();
            doc.add(new Field(FIELD_NORMAL, text, Field.Store.YES, Field.Index.ANALYZED));
            doc.add(new Field(FIELD_AUTOCOMPLETE, text.toLowerCase(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            writer.addDocument(doc);
        }

    }

    @Test
    public void prefixParser() throws ParseException {
        Query prefixQuery = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fre*");
        assertTrue(prefixQuery instanceof PrefixQuery);

        Query normalQuery = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fred");
        assertFalse(normalQuery instanceof PrefixQuery);
    }

    @Test
    public void normal() throws ParseException, IOException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_NORMAL, analyzer).parse("Fred");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(2, topDocs.totalHits);
    }

    @Test
    public void autocomplete() throws IOException, ParseException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Fre*");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(2, topDocs.totalHits);
    }

    @Test
    public void beginningOnly() throws ParseException, IOException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("R*");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(1, topDocs.totalHits);
    }

    @Test
    public void singleTerm() throws ParseException, IOException {
        Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Mado*");
        TopDocs topDocs = searcher.search(query, 10);
        assertEquals(1, topDocs.totalHits);
    }
}

Upvotes: 0

Views: 6753

Answers (1)

jpountz
jpountz

Reputation: 9964

By default, QueryParser lowercases the terms of special queries (in particular prefix queries). To disable this, see QueryParser.setLowercaseExpandedTerms.

Replace

Query query = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer).parse("Mado*");

with

    QueryParser qp = new QueryParser(Version.LUCENE_35, FIELD_AUTOCOMPLETE, analyzer);
    qp.setLowercaseExpandedTerms(false);
    Query query = qp.parse("Mado*");

to fix your tests.

Upvotes: 3

Related Questions