don't fail on nullable locale (currently not forbidden) (#886)
Signed-off-by: Markus Rathgeb <maggu2810@gmail.com>pull/893/head
parent
2ed5110668
commit
7d7f8d044d
|
@ -585,16 +585,17 @@ public abstract class AbstractRuleBasedInterpreter implements HumanLanguageInter
|
||||||
* @return resulting tokens
|
* @return resulting tokens
|
||||||
*/
|
*/
|
||||||
protected ArrayList<String> tokenize(Locale locale, String text) {
|
protected ArrayList<String> tokenize(Locale locale, String text) {
|
||||||
|
final Locale localeSafe = locale != null ? locale : Locale.getDefault();
|
||||||
ArrayList<String> parts = new ArrayList<String>();
|
ArrayList<String> parts = new ArrayList<String>();
|
||||||
if (text == null) {
|
if (text == null) {
|
||||||
return parts;
|
return parts;
|
||||||
}
|
}
|
||||||
String[] split;
|
String[] split;
|
||||||
if ((locale != null) && locale.getLanguage().equalsIgnoreCase(Locale.FRENCH.getLanguage())) {
|
if (localeSafe.getLanguage().equalsIgnoreCase(Locale.FRENCH.getLanguage())) {
|
||||||
split = text.toLowerCase(locale).replaceAll("[\\']", " ").replaceAll("[^\\w\\sàâäçéèêëîïôùûü]", " ")
|
split = text.toLowerCase(localeSafe).replaceAll("[\\']", " ").replaceAll("[^\\w\\sàâäçéèêëîïôùûü]", " ")
|
||||||
.split("\\s");
|
.split("\\s");
|
||||||
} else {
|
} else {
|
||||||
split = text.toLowerCase(locale).replaceAll("[\\']", "").replaceAll("[^\\w\\s]", " ").split("\\s");
|
split = text.toLowerCase(localeSafe).replaceAll("[\\']", "").replaceAll("[^\\w\\s]", " ").split("\\s");
|
||||||
}
|
}
|
||||||
for (int i = 0; i < split.length; i++) {
|
for (int i = 0; i < split.length; i++) {
|
||||||
String part = split[i].trim();
|
String part = split[i].trim();
|
||||||
|
|
Loading…
Reference in New Issue