RobertaTokenizer

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. val appendForPieceId: Option[String]

   

   Attributes

   protected

   Definition Classes

   BpeTokenizer

5. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

6. def bpe(indToken: IndexedToken): Array[TokenPiece]

   

   Do the BPE algorithm.

   Do the BPE algorithm. Goal is to find the token as the largest words in the known vocabulary. If not possible, the word is split into smaller subwords, until they are known.

   returns

   Array of TokenPieces, corresponding to encoded token

   Attributes

   protected

   Definition Classes

   BpeTokenizer

7. val bpeRanks: Map[(String, String), Int]

   

   Attributes

   protected

   Definition Classes

   BpeTokenizer

8. val cache: Map[String, Array[String]]

   

   cache for already encoded tokens

   cache for already encoded tokens

   Attributes

   protected

   Definition Classes

   BpeTokenizer

9. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

10. def encode(indTokens: Array[IndexedToken]): Array[TokenPiece]

    

    Definition Classes

    BpeTokenizer

11. def encode(indToken: IndexedToken): Array[TokenPiece]

    

    Definition Classes

    BpeTokenizer

12. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

13. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

14. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

15. def getBpeRanking: ((String, String)) ⇒ Int

    

    Rankings for the byte pairs.

    Rankings for the byte pairs. Derived from merges.txt

    Attributes

    protected

    Definition Classes

    BpeTokenizer

16. def getBytePairs(word: Array[String]): Array[(String, String)]

    

    Create a sequence of byte-pairs of the word TODO: XLM has to append to end

    Create a sequence of byte-pairs of the word TODO: XLM has to append to end

    Attributes

    protected

    Definition Classes

    BpeTokenizer

17. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

18. def getTokenPieces(indToken: IndexedToken, word: Array[String], processedToken: String): Array[TokenPiece]

    

    Attributes

    protected

    Definition Classes

    BpeTokenizer

19. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

22. final def notify(): Unit

    

    Definition Classes

    AnyRef

23. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

24. def performMerges(wordChars: Array[String], charPairs: Array[(String, String)]): Array[String]

    

    Attributes

    protected

    Definition Classes

    BpeTokenizer

25. def preProcessTokenForBpe(token: String): String

    

    Definition Classes

    RobertaTokenizer → BpeTokenizer

26. val prependForPieceId: Option[String]

    

    Definition Classes

    RobertaTokenizer → BpeTokenizer

27. val sentencePadding: (String, String)

    

    Special tokens of the model for processing

    Special tokens of the model for processing

    Definition Classes

    BpeTokenizer

28. def splitOnSpecialToken(specialToken: SpecialToken, text: String): ListBuffer[String]

    

    Split the the individual sub texts on special tokens, e.g.

    Split the the individual sub texts on special tokens, e.g. masking etc.

    Attributes

    protected

    Definition Classes

    BpeTokenizer

29. val splitPattern: Regex

    

30. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

31. def toString(): String

    

    Definition Classes

    AnyRef → Any

32. def tokenize(sentence: Sentence): Array[IndexedToken]

    

    Tokenize considering special tokens and split algorithm

    Tokenize considering special tokens and split algorithm

    Definition Classes

    BpeTokenizer

33. def tokenizeSubText(text: String, indexOffset: Int): Array[IndexedToken]

    

    Needs to be implemented

    Needs to be implemented

    Definition Classes

    RobertaTokenizer → BpeTokenizer

34. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

35. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

36. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )