scala-transformers-tokenizer/org.clulab.scala_transformers.tokenizer

org.clulab.scala_transformers.tokenizer

package org.clulab.scala_transformers.tokenizer

Type members

Classlikes

case class LongTokenization(tokenIds: Array[Long], wordIds: Array[Long], tokens: Array[String])
Companion:
object
object LongTokenization
Companion:
class
case class Tokenization(tokenIds: Array[Int], wordIds: Array[Int], tokens: Array[String])
abstract class Tokenizer(name: String) extends Tokenizing
trait Tokenizing