don't fail on nullable locale (currently not forbidden) (#886)

Signed-off-by: Markus Rathgeb <maggu2810@gmail.com>
pull/893/head
Markus Rathgeb 2019-06-20 21:34:18 +02:00 committed by Kai Kreuzer
parent 2ed5110668
commit 7d7f8d044d
1 changed files with 4 additions and 3 deletions

View File

@ -585,16 +585,17 @@ public abstract class AbstractRuleBasedInterpreter implements HumanLanguageInter
* @return resulting tokens
*/
protected ArrayList<String> tokenize(Locale locale, String text) {
final Locale localeSafe = locale != null ? locale : Locale.getDefault();
ArrayList<String> parts = new ArrayList<String>();
if (text == null) {
return parts;
}
String[] split;
if ((locale != null) && locale.getLanguage().equalsIgnoreCase(Locale.FRENCH.getLanguage())) {
split = text.toLowerCase(locale).replaceAll("[\\']", " ").replaceAll("[^\\w\\sàâäçéèêëîïôùûü]", " ")
if (localeSafe.getLanguage().equalsIgnoreCase(Locale.FRENCH.getLanguage())) {
split = text.toLowerCase(localeSafe).replaceAll("[\\']", " ").replaceAll("[^\\w\\sàâäçéèêëîïôùûü]", " ")
.split("\\s");
} else {
split = text.toLowerCase(locale).replaceAll("[\\']", "").replaceAll("[^\\w\\s]", " ").split("\\s");
split = text.toLowerCase(localeSafe).replaceAll("[\\']", "").replaceAll("[^\\w\\s]", " ").split("\\s");
}
for (int i = 0; i < split.length; i++) {
String part = split[i].trim();