class JanusTokenizer extends BpeTokenizer

Linear Supertypes
BpeTokenizer, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. JanusTokenizer
  2. BpeTokenizer
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new JanusTokenizer(merges: Map[(String, String), Int], vocab: Map[String, Int], specialTokens: SpecialTokens, padWithSequenceTokens: Boolean = true, prependString: String = "", addPrefixSpaceToSentence: Boolean = false, alwaysAddPrefix: Boolean = true, splitPatternRegex: Regex = raw"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""".r)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val addPrefixSpaceToSentence: Boolean
    Definition Classes
    BpeTokenizer
  5. val alwaysAddPrefix: Boolean
    Definition Classes
    BpeTokenizer
  6. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  7. def bpe(indToken: IndexedToken): Array[TokenPiece]

    Do the BPE algorithm.

    Do the BPE algorithm. Goal is to find the token as the largest words in the known vocabulary. If not possible, the word is split into smaller subwords, until they are known.

    returns

    Array of TokenPieces, corresponding to encoded token

    Attributes
    protected
    Definition Classes
    BpeTokenizer
  8. val bpeRanks: Map[(String, String), Int]
    Attributes
    protected
    Definition Classes
    BpeTokenizer
  9. val bytesToUnicodeMapping: Map[Int, String]

    Mapping for bytes to a different set of unicode characters (especially white spaces).

    Mapping for bytes to a different set of unicode characters (especially white spaces). This improved model performance for gpt-2

    Attributes
    protected
  10. val cache: Map[String, Array[String]]

    cache for already encoded tokens

    cache for already encoded tokens

    Attributes
    protected
    Definition Classes
    BpeTokenizer
  11. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @HotSpotIntrinsicCandidate() @native()
  12. def decodeTokens(tokens: Array[Int]): String
  13. val decoderVocab: Map[Int, String]
    Attributes
    protected
  14. def encode(indTokens: Array[IndexedToken]): Array[TokenPiece]
    Definition Classes
    BpeTokenizer
  15. def encode(indToken: IndexedToken): Array[TokenPiece]
    Definition Classes
    BpeTokenizer
  16. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  17. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  18. def getBpeRanking: ((String, String)) => Int

    Rankings for the byte pairs.

    Rankings for the byte pairs. Derived from merges.txt

    Attributes
    protected
    Definition Classes
    BpeTokenizer
  19. def getBytePairs(word: Array[String]): Array[(String, String)]

    Create a sequence of byte-pairs of the word

    Create a sequence of byte-pairs of the word

    Attributes
    protected
    Definition Classes
    BpeTokenizer
  20. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @HotSpotIntrinsicCandidate() @native()
  21. def getTokenPieces(indToken: IndexedToken, word: Array[String]): Array[TokenPiece]
    Attributes
    protected
    Definition Classes
    BpeTokenizer
  22. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @HotSpotIntrinsicCandidate() @native()
  23. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  24. val merges: Map[(String, String), Int]
    Definition Classes
    BpeTokenizer
  25. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  26. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @HotSpotIntrinsicCandidate() @native()
  27. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @HotSpotIntrinsicCandidate() @native()
  28. val padWithSequenceTokens: Boolean
    Definition Classes
    BpeTokenizer
  29. def performMerges(wordChars: Array[String], charPairs: Array[(String, String)]): Array[String]
    Attributes
    protected
    Definition Classes
    BpeTokenizer
  30. def preProcessTokenForBpe(token: String): String
    Definition Classes
    JanusTokenizer → BpeTokenizer
  31. val prefixForPieceId: Option[String]
    Definition Classes
    JanusTokenizer → BpeTokenizer
  32. val sentencePadding: (String, String)

    Special tokens of the model for processing

    Special tokens of the model for processing

    Definition Classes
    BpeTokenizer
  33. val specialTokens: SpecialTokens
    Definition Classes
    BpeTokenizer
  34. def splitOnSpecialToken(specialToken: SpecialToken, text: String): ListBuffer[String]

    Split the the individual sub texts on special tokens, e.g.

    Split the the individual sub texts on special tokens, e.g. masking etc.

    Attributes
    protected
    Definition Classes
    BpeTokenizer
  35. val splitPattern: Regex
  36. val suffixForPieceId: Option[String]
    Attributes
    protected
    Definition Classes
    BpeTokenizer
  37. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  38. def toString(): String
    Definition Classes
    AnyRef → Any
  39. def tokenize(sentence: Sentence): Array[IndexedToken]

    Tokenize considering special tokens and split algorithm

    Tokenize considering special tokens and split algorithm

    Definition Classes
    BpeTokenizer
  40. def tokenizeSubText(text: String, indexOffset: Int): Array[IndexedToken]

    Needs to be implemented

    Needs to be implemented

    Definition Classes
    JanusTokenizer → BpeTokenizer
  41. val unicodeToByteMapping: Map[String, Int]
    Attributes
    protected
  42. val vocab: Map[String, Int]
    Definition Classes
    BpeTokenizer
  43. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  44. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  45. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from BpeTokenizer

Inherited from AnyRef

Inherited from Any

Ungrouped