-
Notifications
You must be signed in to change notification settings - Fork 84
/
processor.py
204 lines (180 loc) · 7.65 KB
/
processor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
# -*- coding: utf-8 -*-
import re
from pysbd.utils import Text
from pysbd.lists_item_replacer import ListItemReplacer
from pysbd.exclamation_words import ExclamationWords
from pysbd.between_punctuation import BetweenPunctuation
from pysbd.abbreviation_replacer import AbbreviationReplacer
class Processor(object):
def __init__(self, text, lang, char_span=False):
"""Process a text - do pre and post processing - to get proper sentences
Parameters
----------
text : str
Original text
language : object
Language module
char_span : bool, optional
Get start & end character offsets of each sentences
within original text, by default False
"""
self.text = text
self.lang = lang
self.char_span = char_span
def process(self):
if not self.text:
return self.text
self.text = self.text.replace('\n', '\r')
li = ListItemReplacer(self.text)
self.text = li.add_line_break()
self.replace_abbreviations()
self.replace_numbers()
self.replace_continuous_punctuation()
self.replace_periods_before_numeric_references()
self.text = Text(self.text).apply(
self.lang.Abbreviation.WithMultiplePeriodsAndEmailRule,
self.lang.GeoLocationRule, self.lang.FileFormatRule)
postprocessed_sents = self.split_into_segments()
return postprocessed_sents
def rm_none_flatten(self, sents):
"""Remove None values and unpack list of list sents
Parameters
----------
sents : list
list of sentences
Returns
-------
list
unpacked and None removed list of sents
"""
sents = list(filter(None, sents))
if not any(isinstance(s, list) for s in sents):
return sents
new_sents = []
for sent in sents:
if isinstance(sent, list):
for s in sent:
new_sents.append(s)
else:
new_sents.append(sent)
return new_sents
def split_into_segments(self):
self.check_for_parens_between_quotes()
sents = self.text.split('\r')
# remove empty and none values
sents = self.rm_none_flatten(sents)
sents = [
Text(s).apply(self.lang.SingleNewLineRule, *self.lang.EllipsisRules.All)
for s in sents
]
sents = [self.check_for_punctuation(s) for s in sents]
# flatten list of list of sentences
sents = self.rm_none_flatten(sents)
postprocessed_sents = []
for sent in sents:
sent = Text(sent).apply(*self.lang.SubSymbolsRules.All)
post_process_sent = self.post_process_segments(sent)
if post_process_sent and isinstance(post_process_sent, str):
postprocessed_sents.append(post_process_sent)
elif isinstance(post_process_sent, list):
for pps in post_process_sent:
postprocessed_sents.append(pps)
postprocessed_sents = [Text(ns).apply(self.lang.SubSingleQuoteRule)
for ns in postprocessed_sents]
return postprocessed_sents
def post_process_segments(self, txt):
if len(txt) > 2 and re.search(r'\A[a-zA-Z]*\Z', txt):
return txt
# below condition present in pragmatic segmenter
# dont know significance of it yet.
# if self.consecutive_underscore(txt) or len(txt) < 2:
# return txt
if re.match(r'\t', txt):
pass
# TODO:
# Decide on keeping or removing Standard.ExtraWhiteSpaceRule
# removed to retain original text spans
# txt = Text(txt).apply(*ReinsertEllipsisRules.All,
# Standard.ExtraWhiteSpaceRule)
txt = Text(txt).apply(*self.lang.ReinsertEllipsisRules.All)
if re.search(self.lang.QUOTATION_AT_END_OF_SENTENCE_REGEX, txt):
txt = re.split(
self.lang.SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX, txt)
return txt
else:
txt = txt.replace('\n', '')
return txt.strip()
def check_for_parens_between_quotes(self):
def paren_replace(match):
match = match.group()
sub1 = re.sub(r'\s(?=\()', '\r', match)
sub2 = re.sub(r'(?<=\))\s', '\r', sub1)
return sub2
self.text = re.sub(self.lang.PARENS_BETWEEN_DOUBLE_QUOTES_REGEX,
paren_replace, self.text)
def replace_continuous_punctuation(self):
def continuous_puncs_replace(match):
match = match.group()
sub1 = re.sub(re.escape('!'), '&ᓴ&', match)
sub2 = re.sub(re.escape('?'), '&ᓷ&', sub1)
return sub2
self.text = re.sub(self.lang.CONTINUOUS_PUNCTUATION_REGEX,
continuous_puncs_replace, self.text)
def replace_periods_before_numeric_references(self):
# https://github.com/diasks2/pragmatic_segmenter/commit/d9ec1a352aff92b91e2e572c30bb9561eb42c703
self.text = re.sub(self.lang.NUMBERED_REFERENCE_REGEX,
r"∯\2\r\7", self.text)
def consecutive_underscore(self, txt):
# Rubular: http://rubular.com/r/fTF2Ff3WBL
txt = re.sub(r'_{3,}', '', txt)
return len(txt) == 0
def check_for_punctuation(self, txt):
if any(p in txt for p in self.lang.Punctuations):
sents = self.process_text(txt)
return sents
else:
# NOTE: next steps of check_for_punctuation will unpack this list
return [txt]
def process_text(self, txt):
if txt[-1] not in self.lang.Punctuations:
txt += 'ȸ'
txt = ExclamationWords.apply_rules(txt)
txt = self.between_punctuation(txt)
# handle text having only doublepunctuations
if not re.match(self.lang.DoublePunctuationRules.DoublePunctuation, txt):
txt = Text(txt).apply(*self.lang.DoublePunctuationRules.All)
txt = Text(txt).apply(self.lang.QuestionMarkInQuotationRule,
*self.lang.ExclamationPointRules.All)
txt = ListItemReplacer(txt).replace_parens()
txt = self.sentence_boundary_punctuation(txt)
return txt
def replace_numbers(self):
self.text = Text(self.text).apply(*self.lang.Numbers.All)
def abbreviations_replacer(self):
if hasattr(self.lang, "AbbreviationReplacer"):
return self.lang.AbbreviationReplacer(self.text, self.lang)
else:
return AbbreviationReplacer(self.text, self.lang)
def replace_abbreviations(self):
self.text = self.abbreviations_replacer().replace()
def between_punctuation_processor(self, txt):
if hasattr(self.lang, "BetweenPunctuation"):
return self.lang.BetweenPunctuation(txt)
else:
return BetweenPunctuation(txt)
def between_punctuation(self, txt):
txt = self.between_punctuation_processor(txt).replace()
return txt
def sentence_boundary_punctuation(self, txt):
if hasattr(self.lang, 'ReplaceColonBetweenNumbersRule'):
txt = Text(txt).apply(
self.lang.ReplaceColonBetweenNumbersRule)
if hasattr(self.lang, 'ReplaceNonSentenceBoundaryCommaRule'):
txt = Text(txt).apply(
self.lang.ReplaceNonSentenceBoundaryCommaRule)
# retain exclamation mark if it is an ending character of a given text
txt = re.sub(r'&ᓴ&$', '!', txt)
txt = [
m.group() for m in re.finditer(self.lang.SENTENCE_BOUNDARY_REGEX, txt)
]
return txt