package tokenizerapi

//go:generate mockery --name=Tokenizer --with-expecter
type Tokenizer interface {
	NewTokenStream(text string) TokenStream
	Clone() (Tokenizer, error)
	Destroy()
}