1: <?php
  2: 
  3: /**
  4:  * Segments sentences.
  5:  * Clipping may not be perfect.
  6:  * Sentence count should be VERY close to the truth.
  7:  * 
  8:  * Multibyte safe (atleast for UTF-8), but rules based on germanic
  9:  * language stucture (English, Dutch, German). Should work for most
 10:  * latin-alphabet languages.
 11:  */
 12: class Sentence {
 13:     /**
 14:      * Specify this flag with the split method to trim whitespace.
 15:      */
 16:     const SPLIT_TRIM        = 0x1;
 17: 
 18:     /**
 19:      * List of characters used to terminate sentences.
 20:      * @var array
 21:      */
 22:     private $terminals      = array('.', '!', '?');
 23: 
 24:     /**
 25:      * List of characters used for abbreviations.
 26:      * @var array
 27:      */
 28:     private $abbreviators   = array('.');
 29: 
 30:     /**
 31:      * Multibyte safe version of standard trim() function.
 32:      * @param string $string
 33:      * @return string
 34:      */
 35:     private static function mbTrim($string) {
 36:         return mb_ereg_replace('^\s*([\s\S]*?)\s*$', '\1', $string);
 37:     }
 38: 
 39:     /**
 40:      * A cross between mb_split and preg_split, adding the preg_split flags
 41:      * to mb_split.
 42:      * @param string $pattern
 43:      * @param string $string
 44:      * @param int $limit
 45:      * @param int $flags
 46:      * @return array
 47:      */
 48:     private static function mbSplit($pattern, $string, $limit = -1, $flags = 0) {       
 49:         $strlen = strlen($string);      // bytes!   
 50:         mb_ereg_search_init($string);
 51:         
 52:         $lengths = array();
 53:         $position = 0;
 54:         while (($array = mb_ereg_search_pos($pattern, '')) !== false) {
 55:             // capture split
 56:             $lengths[] = array($array[0] - $position, false, null);
 57:                     
 58:             // move position
 59:             $position = $array[0] + $array[1];
 60:         
 61:             // capture delimiter
 62:             $regs = mb_ereg_search_getregs();           
 63:             $lengths[] = array($array[1], true, isset($regs[1]) && $regs[1]);
 64:             
 65:             // Continue on?
 66:             if ($position >= $strlen) {
 67:                 break;
 68:             }           
 69:         }
 70: 
 71:         // Add last bit, if not ending with split
 72:         $lengths[] = array($strlen - $position, false, null);
 73: 
 74:         // Substrings
 75:         $parts = array();
 76:         $position = 0;      
 77:         $count = 1;
 78:         foreach ($lengths as $length) {
 79:             $is_delimiter   = $length[1];
 80:             $is_captured    = $length[2];
 81:             
 82:             if ($limit > 0 && !$is_delimiter && ($length[0] || ~$flags & PREG_SPLIT_NO_EMPTY) && ++$count > $limit) {
 83:                 if ($length[0] > 0 || ~$flags & PREG_SPLIT_NO_EMPTY) {          
 84:                     $parts[]    = $flags & PREG_SPLIT_OFFSET_CAPTURE
 85:                                 ? array(mb_strcut($string, $position), $position)
 86:                                 : mb_strcut($string, $position);                
 87:                 }
 88:                 break;
 89:             } elseif ((!$is_delimiter || ($flags & PREG_SPLIT_DELIM_CAPTURE && $is_captured))
 90:                    && ($length[0] || ~$flags & PREG_SPLIT_NO_EMPTY)) {
 91:                 $parts[]    = $flags & PREG_SPLIT_OFFSET_CAPTURE
 92:                             ? array(mb_strcut($string, $position, $length[0]), $position)
 93:                             : mb_strcut($string, $position, $length[0]);
 94:             }
 95:             
 96:             $position += $length[0];
 97:         }
 98:         
 99:         return $parts;
100:     }   
101:     
102:     /**
103:      * Breaks a piece of text into lines by linebreak.
104:      * Eats up any linebreak characters as if one.
105:      * 
106:      * Multibyte safe
107:      * 
108:      * @param string $text
109:      * @return array
110:      */
111:     private static function linebreakSplit($text) {
112:         $lines = array();
113:         $line = '';
114:         
115:         foreach (self::mbSplit('([\r\n]+)', $text, -1, PREG_SPLIT_DELIM_CAPTURE) as $part) {
116:             $line .= $part;
117:             if (self::mbTrim($part) === '') {
118:                 $lines[] = $line;
119:                 $line = '';
120:             }
121:         }
122:         $lines[] = $line;
123:         
124:         return $lines;
125:     }
126: 
127:     /**
128:      * Splits an array of lines by (consecutive sequences of)
129:      * terminals, keeping terminals.
130:      * 
131:      * Multibyte safe (atleast for UTF-8)
132:      * 
133:      * For example:
134:      *  "There ... is. More!"
135:      *      ... becomes ...
136:      *  [ "There ", "...", " is", ".", " More", "!" ]
137:      * 
138:      * @param array $lines
139:      * @return array
140:      */
141:     private function punctuationSplit($line) {                                      
142:         $parts = array();
143: 
144:         $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY); // This is UTF8 multibyte safe!
145:         $is_terminal = in_array($chars[0], $this->terminals);
146:         
147:         $part = '';
148:         foreach ($chars as $index => $char) {
149:             if (in_array($char, $this->terminals) !== $is_terminal) {
150:                 $parts[] = $part;
151:                 $part = '';
152:                 $is_terminal = !$is_terminal;
153:             }
154:             $part .= $char;                         
155:         }
156:         
157:         if (!empty($part)) {
158:             $parts[] = $part;                           
159:         }
160: 
161:         return $parts;
162:     }
163: 
164:     /**
165:      * Appends each terminal item after it's preceding
166:      * non-terminals.
167:      * 
168:      * Multibyte safe (atleast for UTF-8)
169:      * 
170:      * For example:
171:      *  [ "There ", "...", " is", ".", " More", "!" ]
172:      *      ... becomes ...
173:      *  [ "There ... is.", "More!" ]
174:      * 
175:      * @param array $punctuations
176:      * @return array
177:      */
178:     private function punctuationMerge($punctuations) {      
179:         $definite_terminals = array_diff($this->terminals, $this->abbreviators);
180:         
181:         $merges = array();
182:         $merge = '';
183: 
184:         foreach ($punctuations as $punctuation) {
185:             if ($punctuation !== '') {
186:                 $merge.= $punctuation;
187:                 if (mb_strlen($punctuation) === 1 && in_array($punctuation, $this->terminals)) {
188:                     $merges[] = $merge;
189:                     $merge = '';
190:                 } else {
191:                     foreach ($definite_terminals as $terminal) {
192:                         if (mb_strpos($punctuation, $terminal) !== false) {
193:                             $merges[] = $merge;
194:                             $merge = '';
195:                             break;
196:                         }
197:                     }
198:                 }
199:             }           
200:         }
201:         if (!empty($merge)) {
202:             $merges[] = $merge;
203:         }
204: 
205:         return $merges;
206:     }
207: 
208:     /**
209:      * Merges any one-word items with it's preceding items.
210:      * 
211:      * Multibyte safe
212:      * 
213:      * For example:
214:      *  [ "There ... is.", "More!" ]
215:      *      ... becomes ...
216:      *  [ "There ... is. More!" ]
217:      * 
218:      * @param array $fragments
219:      * @return array
220:      */
221:     private function abbreviationMerge($fragments) {
222:         $non_abbreviating_terminals = array_diff($this->terminals, $this->abbreviators);
223:         
224:         $abbreviations = array();
225:         
226:         $abbreviation = '';
227:         
228:         $previous_word_count = null;
229:         $previous_word_ending = null;       
230:         foreach ($fragments as $fragment) {
231:             $word_count = count(mb_split('\s+', self::mbTrim($fragment)));
232:             $starts_with_space = mb_ereg_match('^\s+', $fragment);          
233:             $after_non_abbreviating_terminal = in_array($previous_word_ending, $non_abbreviating_terminals);
234:             
235:             if ($after_non_abbreviating_terminal || ($previous_word_count !== null && ($previous_word_count !== 1 || $word_count !== 1) && $starts_with_space)) {
236:                 $abbreviations[] = $abbreviation;
237:                 $abbreviation = '';
238:             }
239: 
240:             $abbreviation           .= $fragment;                   
241:             $previous_word_count    = $word_count;                          
242:             $previous_word_ending   = mb_substr($fragment, -1);         
243:         }
244:         if ($abbreviation !== '') {
245:             $abbreviations[] = $abbreviation;
246:         }
247: 
248:         return $abbreviations;
249:     }
250: 
251:     /**
252:      * Merges items into larger sentences.
253:      * 
254:      * Multibyte safe
255:      * 
256:      * @param array $shorts
257:      * @return array
258:      */
259:     private function sentenceMerge($shorts) {
260:         $non_abbreviating_terminals = array_diff($this->terminals, $this->abbreviators);
261: 
262:         $sentences = array();
263: 
264:         $sentence = '';                 
265:         $has_words = false;
266:         $previous_word_ending = null;
267:         foreach ($shorts as $short) {
268:             $word_count = count(mb_split('\s+', self::mbTrim($short)));         
269:             $after_non_abbreviating_terminal = in_array($previous_word_ending, $non_abbreviating_terminals);
270:             
271:             if ($after_non_abbreviating_terminal || ($has_words && $word_count > 1)) {
272:                 $sentences[] = $sentence;
273:                 $sentence = '';                     
274:                 $has_words = $word_count > 1;
275:             } else {
276:                 $has_words = $has_words || $word_count > 1;                     
277:             }
278:             
279:             $sentence.= $short;         
280:             $previous_word_ending = mb_substr($short, -1);                  
281:         }
282:         if (!empty($sentence)) {
283:             $sentences[] = $sentence;
284:         }           
285: 
286:         return $sentences;
287:     }
288: 
289:     /**
290:      * Return the sentences sentences detected in the provided text.
291:      * Set the Sentence::SPLIT_TRIM flag to trim whitespace.
292:      * @param string $text
293:      * @param integer $flags
294:      * @return array
295:      */
296:     public function split($text, $flags = 0) {      
297:         $sentences = array();
298: 
299:         // Split
300:         foreach (self::linebreakSplit($text) as $line) {                
301:             if (self::mbTrim($line) !== '') {
302:                 $punctuations   = $this->punctuationSplit($line);
303:                 $merges         = $this->punctuationMerge($punctuations);
304:                 $shorts         = $this->abbreviationMerge($merges);
305:                 $sentences      = array_merge($sentences, $this->sentenceMerge($shorts));
306:             }
307:         }
308:         
309:         // Post process
310:         if ($flags & self::SPLIT_TRIM) {
311:             foreach ($sentences as &$sentence) {
312:                 $sentence = self::mbTrim($sentence);
313:             }
314:             unset($sentence);
315:         }
316: 
317:         return $sentences;
318:     }
319: 
320:     /**
321:      * Return the number of sentences detected in the provided text.
322:      * @param string $text
323:      * @return integer
324:      */
325:     public function count($text) {
326:         return count($this->split($text));
327:     }
328: }