milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: empty analyzer params not use standard tokenizer (#38148) 

relate: https://github.com/milvus-io/milvus/issues/35853

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
pull/38212/head
aoiasd 2024-12-04 14:58:39 +08:00 committed by GitHub
parent 1f66b9ebfb
commit 87aa9a0f2d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
internal/core/thirdparty/tantivy/tantivy-binding/src/tokenizer.rs vendored
View File

@ -87,10 +87,12 @@ impl AnalyzerBuilder<'_> {
}
}
fn get_tokenizer_name(&self) -> Result<String> {
let tokenizer = self.params.get("tokenizer");
if tokenizer.is_none() {
return Ok("standard".to_string());
fn get_tokenizer_name(&self) -> Result<String>{
let tokenizer=self.params.get("tokenizer");
if tokenizer.is_none(){
return Err(TantivyBindingError::InternalError(format!(
"tokenizer name or type must be set"
)));
}
if !tokenizer.unwrap().is_string() {
return Err(TantivyBindingError::InternalError(format!(
@ -257,8 +259,14 @@ pub(crate) fn create_tokenizer_with_filter(params: &String) -> Result<TextAnalyz
"analyzer params should be a json map".to_string(),
));
}
let mut builder = AnalyzerBuilder::new(analyzer_params.unwrap().as_object().unwrap());
let builder_params = analyzer_params.unwrap().as_object().unwrap();
if builder_params.is_empty(){
return Ok(standard_analyzer(vec![]));
}
let mut builder = AnalyzerBuilder::new(builder_params);
// build custom filter
let filter_params = json_params.get("filter");
if !filter_params.is_none() && filter_params.unwrap().is_object() {
@ -282,7 +290,6 @@ pub(crate) fn create_tokenizer(params: &String) -> Result<TextAnalyzer> {
#[cfg(test)]
mod tests {
use crate::tokenizer::create_tokenizer;
use regex;
#[test]
fn test_standard_analyzer() {