package bpe
Ordering
- Alphabetic
Visibility
- Public
- All
Type Members
- class Gpt2Tokenizer extends BpeTokenizer
- class RobertaTokenizer extends Gpt2Tokenizer
- case class SpecialToken(content: String, id: Int, singleWord: Boolean = false, lstrip: Boolean = false, rstrip: Boolean = false) extends Product with Serializable
Value Members
- object BpeTokenizer