package tokenizerapi

import "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"

//go:generate mockery --name=TokenStream --with-expecter
type TokenStream interface {
	Advance() bool
	Token() string
	DetailedToken() *milvuspb.AnalyzerToken
	Destroy()
}