CODEWITHSUNDEEP
pythonnlpspacy
Harsh Trivedi
Harsh Trivedi

Reputation: 1624

Custom sentence segmentation in Spacy

I want spaCy to use the sentence segmentation boundaries as I provide instead of its own processing.

For example:

get_sentences("Bob meets Alice. @SentBoundary@ They play together.")
# => ["Bob meets Alice.", "They play together."]  # two sents

get_sentences("Bob meets Alice. They play together.")
# => ["Bob meets Alice. They play together."]  # ONE sent

get_sentences("Bob meets Alice, @SentBoundary@ they play together.")
# => ["Bob meets Alice,", "they play together."] # two sents

This is what I have so far (borrowing things from documentation here):

import spacy
nlp = spacy.load('en_core_web_sm')

def mark_sentence_boundaries(doc):
    for i, token in enumerate(doc):
        if token.text == '@SentBoundary@':
            doc[i+1].sent_start = True
    return doc

nlp.add_pipe(mark_sentence_boundaries, before='parser')

def get_sentences(text):
    doc = nlp(text)
    return (list(doc.sents))

But the results I get are as follows:

# Ex1
get_sentences("Bob meets Alice. @SentBoundary@ They play together.")
#=> ["Bob meets Alice.", "@SentBoundary@", "They play together."]

# Ex2
get_sentences("Bob meets Alice. They play together.")
#=> ["Bob meets Alice.", "They play together."]

# Ex3
get_sentences("Bob meets Alice, @SentBoundary@ they play together.")
#=> ["Bob meets Alice, @SentBoundary@", "they play together."]

Following are main problems I am facing:

  1. When sentence break is found, how to get rid of @SentBoundary@ token.
  2. How to disallow spaCy from splitting if @SentBoundary@ is not present.

Upvotes: 6

Views: 5906

Answers (2)

Akoffice
Akoffice

Reputation: 381

import spacy
from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA
from spacy.tokens import Doc
import numpy
nlp = spacy.load('en_core_web_sm')

def mark_sentence_boundaries(doc):
    indexes = []
    for i, token in enumerate(doc):
        if token.text == '@SentBoundary@':
            doc[i+1].sent_start = True
            indexes.append(token.i)

    np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA])
    np_array = numpy.delete(np_array, indexes, axis=0)
    doc2 = Doc(doc.vocab, words=[t.text for i, t in enumerate(doc) if i not in indexes])
    doc2.from_array([LOWER, POS, ENT_TYPE, IS_ALPHA], np_array)
    return doc2

nlp.add_pipe(mark_sentence_boundaries, before='parser')

def get_sentences(text):
    doc = nlp(text)
    return (list(doc.sents))

print(get_sentences("Bob meets Alice. @SentBoundary@ They play together."))
# => ["Bob meets Alice.", "They play together."]  # two sents

print(get_sentences("Bob meets Alice. They play together."))
# => ["Bob meets Alice. They play together."]  # ONE sent

print(get_sentences("Bob meets Alice, @SentBoundary@ they play together."))
# => ["Bob meets Alice,", "they play together."] # two sents

Upvotes: 0

Harsh Trivedi
Harsh Trivedi

Reputation: 1624

The following code works:

import spacy
nlp = spacy.load('en_core_web_sm')

def split_on_breaks(doc):
    start = 0
    seen_break = False
    for word in doc:
        if seen_break:
            yield doc[start:word.i-1]
            start = word.i
            seen_break = False
        elif word.text == '@SentBoundary@':
            seen_break = True
    if start < len(doc):
        yield doc[start:len(doc)]

sbd = SentenceSegmenter(nlp.vocab, strategy=split_on_breaks)
nlp.add_pipe(sbd, first=True)

def get_sentences(text):
    doc = nlp(text)
    return (list(doc.sents)) # convert to string if required.

# Ex1
get_sentences("Bob meets Alice. @SentBoundary@ They play together.")
# => ["Bob meets Alice.", "They play together."]  # two sentences

# Ex2
get_sentences("Bob meets Alice. They play together.")
# => ["Bob meets Alice. They play together."]  # ONE sentence

# Ex3
get_sentences("Bob meets Alice, @SentBoundary@ they play together.")
# => ["Bob meets Alice,", "they play together."] # two sentences

Right thing was to check for SentenceSegmenter than manual boundary setting (examples here). This github issue was also helpful.

Upvotes: 7

Related Questions