Tokenizer

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def $[T](param: Param[T]): T

   

   Attributes

   protected

   Definition Classes

   Params

4. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

5. def addContextChars(v: String): Tokenizer.this.type

   

6. def addException(value: String): Tokenizer.this.type

   

7. def addInfixPattern(value: String): Tokenizer.this.type

   

8. def addSplitChars(v: String): Tokenizer.this.type

   

9. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

10. def beforeTraining(spark: SparkSession): Unit

    

    Definition Classes

    AnnotatorApproach

11. def buildRuleFactory: RuleFactory

    

12. val caseSensitiveExceptions: BooleanParam

    

13. final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

    

    Attributes

    protected

    Definition Classes

    HasInputAnnotationCols

14. final def clear(param: Param[_]): Tokenizer.this.type

    

    Definition Classes

    Params

15. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

16. val contextChars: StringArrayParam

    

17. final def copy(extra: ParamMap): Estimator[TokenizerModel]

    

    Definition Classes

    AnnotatorApproach → Estimator → PipelineStage → Params

18. def copyValues[T <: Params](to: T, extra: ParamMap): T

    

    Attributes

    protected

    Definition Classes

    Params

19. final def defaultCopy[T <: Params](extra: ParamMap): T

    

    Attributes

    protected

    Definition Classes

    Params

20. val description: String

    

    Definition Classes

    Tokenizer → AnnotatorApproach

21. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

22. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

23. val exceptions: StringArrayParam

    

24. val exceptionsPath: ExternalResourceParam

    

25. def explainParam(param: Param[_]): String

    

    Definition Classes

    Params

26. def explainParams(): String

    

    Definition Classes

    Params

27. final def extractParamMap(): ParamMap

    

    Definition Classes

    Params

28. final def extractParamMap(extra: ParamMap): ParamMap

    

    Definition Classes

    Params

29. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

30. final def fit(dataset: Dataset[_]): TokenizerModel

    

    Definition Classes

    AnnotatorApproach → Estimator

31. def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[TokenizerModel]

    

    Definition Classes

    Estimator

    Annotations

    @Since( "2.0.0" )

32. def fit(dataset: Dataset[_], paramMap: ParamMap): TokenizerModel

    

    Definition Classes

    Estimator

    Annotations

    @Since( "2.0.0" )

33. def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): TokenizerModel

    

    Definition Classes

    Estimator

    Annotations

    @Since( "2.0.0" ) @varargs()

34. final def get[T](param: Param[T]): Option[T]

    

    Definition Classes

    Params

35. def getCaseSensitiveExceptions(value: Boolean): Boolean

    

36. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

37. def getContextChars: Array[String]

    

38. final def getDefault[T](param: Param[T]): Option[T]

    

    Definition Classes

    Params

39. def getExceptions: Array[String]

    

40. def getInfixPatterns: Array[String]

    

41. def getInputCols: Array[String]

    
    returns

    input annotations columns currently used

    Definition Classes

    HasInputAnnotationCols

42. def getMaxLength(value: Int): Int

    

43. def getMinLength(value: Int): Int

    

44. final def getOrDefault[T](param: Param[T]): T

    

    Definition Classes

    Params

45. final def getOutputCol: String

    

    Gets annotation column name going to generate

    Gets annotation column name going to generate

    Definition Classes

    HasOutputAnnotationCol

46. def getParam(paramName: String): Param[Any]

    

    Definition Classes

    Params

47. def getPrefixPattern: String

    

48. def getSplitChars: Array[String]

    

49. def getSuffixPattern: String

    

50. def getTargetPattern: String

    

51. final def hasDefault[T](param: Param[T]): Boolean

    

    Definition Classes

    Params

52. def hasParam(paramName: String): Boolean

    

    Definition Classes

    Params

53. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

54. val infixPatterns: StringArrayParam

    

55. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

56. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

57. val inputAnnotatorTypes: Array[AnnotatorType]

    

    A Tokenizer could require only for now a SentenceDetector annotator

    A Tokenizer could require only for now a SentenceDetector annotator

    Definition Classes

    Tokenizer → HasInputAnnotationCols

58. final val inputCols: StringArrayParam

    

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    Attributes

    protected

    Definition Classes

    HasInputAnnotationCols

59. final def isDefined(param: Param[_]): Boolean

    

    Definition Classes

    Params

60. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

61. final def isSet(param: Param[_]): Boolean

    

    Definition Classes

    Params

62. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

63. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

64. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

65. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

66. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

67. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

68. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

69. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

70. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

71. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

72. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

73. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

74. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

75. val maxLength: IntParam

    

76. val minLength: IntParam

    

77. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

78. final def notify(): Unit

    

    Definition Classes

    AnyRef

79. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

80. def onTrained(model: TokenizerModel, spark: SparkSession): Unit

    

    Definition Classes

    AnnotatorApproach

81. val outputAnnotatorType: AnnotatorType

    

    Definition Classes

    Tokenizer → HasOutputAnnotatorType

82. final val outputCol: Param[String]

    

    Attributes

    protected

    Definition Classes

    HasOutputAnnotationCol

83. lazy val params: Array[Param[_]]

    

    Definition Classes

    Params

84. val prefixPattern: Param[String]

    

85. def save(path: String): Unit

    

    Definition Classes

    MLWritable

    Annotations

    @Since( "1.6.0" ) @throws( ... )

86. final def set(paramPair: ParamPair[_]): Tokenizer.this.type

    

    Attributes

    protected

    Definition Classes

    Params

87. final def set(param: String, value: Any): Tokenizer.this.type

    

    Attributes

    protected

    Definition Classes

    Params

88. final def set[T](param: Param[T], value: T): Tokenizer.this.type

    

    Definition Classes

    Params

89. def setCaseSensitiveExceptions(value: Boolean): Tokenizer.this.type

    

90. def setContextChars(v: Array[String]): Tokenizer.this.type

    

91. final def setDefault(paramPairs: ParamPair[_]*): Tokenizer.this.type

    

    Attributes

    protected

    Definition Classes

    Params

92. final def setDefault[T](param: Param[T], value: T): Tokenizer.this.type

    

    Attributes

    protected

    Definition Classes

    Params

93. def setExceptions(value: Array[String]): Tokenizer.this.type

    

94. def setExceptionsPath(path: String, readAs: Format = ReadAs.TEXT, options: Map[String, String] = Map("format" -> "text")): Tokenizer.this.type

    

95. def setInfixPatterns(value: Array[String]): Tokenizer.this.type

    

96. final def setInputCols(value: String*): Tokenizer.this.type

    

    Definition Classes

    HasInputAnnotationCols

97. final def setInputCols(value: Array[String]): Tokenizer.this.type

    

    Overrides required annotators column if different than default

    Overrides required annotators column if different than default

    Definition Classes

    HasInputAnnotationCols

98. def setMaxLength(value: Int): Tokenizer.this.type

    

99. def setMinLength(value: Int): Tokenizer.this.type

    

100. final def setOutputCol(value: String): Tokenizer.this.type

     

     Overrides annotation column name when transforming

     Overrides annotation column name when transforming

     Definition Classes

     HasOutputAnnotationCol

101. def setPrefixPattern(value: String): Tokenizer.this.type

     

102. def setSplitChars(v: Array[String]): Tokenizer.this.type

     

103. def setSuffixPattern(value: String): Tokenizer.this.type

     

104. def setTargetPattern(value: String): Tokenizer.this.type

     

105. val splitChars: StringArrayParam

     

106. val suffixPattern: Param[String]

     

107. final def synchronized[T0](arg0: ⇒ T0): T0

     

     Definition Classes

     AnyRef

108. val targetPattern: Param[String]

     

109. def toString(): String

     

     Definition Classes

     Identifiable → AnyRef → Any

110. def train(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): TokenizerModel

     

     Definition Classes

     Tokenizer → AnnotatorApproach

111. final def transformSchema(schema: StructType): StructType

     

     requirement for pipeline transformation validation.

     requirement for pipeline transformation validation. It is called on fit()

     Definition Classes

     AnnotatorApproach → PipelineStage

112. def transformSchema(schema: StructType, logging: Boolean): StructType

     

     Attributes

     protected

     Definition Classes

     PipelineStage

     Annotations

     @DeveloperApi()

113. val uid: String

     

     Definition Classes

     Tokenizer → Identifiable

114. def validate(schema: StructType): Boolean

     

     takes a Dataset and checks to see if all the required annotation types are present.

     takes a Dataset and checks to see if all the required annotation types are present.

     schema

     to be validated

     returns

     True if all the required types are present, else false

     Attributes

     protected

     Definition Classes

     AnnotatorApproach

115. final def wait(): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

116. final def wait(arg0: Long, arg1: Int): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

117. final def wait(arg0: Long): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

118. def write: MLWriter

     

     Definition Classes

     DefaultParamsWritable → MLWritable