CODEWITHSUNDEEP
javalucene
qqilihq
qqilihq

Reputation: 11474

Getting term counts in Lucene 4 index

I want to get the number of occurrences for each term in a Lucene 4 index (there are some examples on how to do that with older Lucene versions, but as we know, they totally change their API with each new revision, so those how-tos do not help me at all).

What I have now:

IndexReader indexReader = DirectoryReader.open(directory);
Fields fields = MultiFields.getFields(indexReader);
for (String field : fields) {
    Terms terms = fields.terms(field);
    TermsEnum termEnum = terms.iterator(null);
    BytesRef bytesRef;
    while ((bytesRef = termEnum.next()) != null) {
        // what to do here? I have a BytesRef and there is 
        // IndexReader#docFreq(term) but it requires a Term
    }
}

Upvotes: 1

Views: 4037

Answers (1)

slckin
slckin

Reputation: 929

    FSDirectory directory = FSDirectory.open(new File("/tmp/moo"));
    /*
    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_44, new StandardAnalyzer(Version.LUCENE_44)));
    Document document = new Document();
    document.add(new TextField("foo", "abc", Store.YES));
    document.add(new TextField("foo", "abc", Store.YES));
    document.add(new TextField("foo", "aaa", Store.YES));
    document.add(new TextField("bar", "abc", Store.YES));
    writer.addDocument(document);
    writer.commit();
    writer.close(true);
    */

    IndexReader indexReader = DirectoryReader.open(directory);

    Bits liveDocs = MultiFields.getLiveDocs(indexReader);
    Fields fields = MultiFields.getFields(indexReader);
    for (String field : fields) {

        TermsEnum termEnum = MultiFields.getTerms(indexReader, field).iterator(null);
        BytesRef bytesRef;
        while ((bytesRef = termEnum.next()) != null) {
            if (termEnum.seekExact(bytesRef, true)) {

                DocsEnum docsEnum = termEnum.docs(liveDocs, null);

                if (docsEnum != null) {
                    int doc;
                    while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                        System.out.println(bytesRef.utf8ToString() + " in doc " + doc + ": " + docsEnum.freq());
                    }
                }
            }
        }
    }

    for (String field : fields) {
        TermsEnum termEnum = MultiFields.getTerms(indexReader, field).iterator(null);
        BytesRef bytesRef;
        while ((bytesRef = termEnum.next()) != null) {
            int freq = indexReader.docFreq(new Term(field, bytesRef));

            System.out.println(bytesRef.utf8ToString() + " in " + freq + " documents");

        }
    }

Upvotes: 4

Related Questions