milvus/internal/util/tokenizerapi/token_stream.go

package tokenizerapi

//go:generate mockery --name=TokenStream --with-expecter
type TokenStream interface {
	Advance() bool
	Token() string
	Destroy()
}