NerOverwriter

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def $[T](param: Param[T]): T

   

   Attributes

   protected

   Definition Classes

   Params

4. def $$[T](feature: StructFeature[T]): T

   

   Attributes

   protected

   Definition Classes

   HasFeatures

5. def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

   

   Attributes

   protected

   Definition Classes

   HasFeatures

6. def $$[T](feature: SetFeature[T]): Set[T]

   

   Attributes

   protected

   Definition Classes

   HasFeatures

7. def $$[T](feature: ArrayFeature[T]): Array[T]

   

   Attributes

   protected

   Definition Classes

   HasFeatures

8. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

9. def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

   

   Attributes

   protected

   Definition Classes

   AnnotatorModel

10. def afterAnnotate(dataset: DataFrame): DataFrame

    

    Attributes

    protected

    Definition Classes

    AnnotatorModel

11. def annotate(annotations: Seq[Annotation]): Seq[Annotation]

    

    takes a document and annotations and produces new annotations of this annotator's annotation type

    takes a document and annotations and produces new annotations of this annotator's annotation type

    annotations

    Annotations that correspond to inputAnnotationCols generated by previous annotators if any

    returns

    any number of annotations processed for every input annotation. Not necessary one to one relationship

    Definition Classes

    NerOverwriter → HasSimpleAnnotate

12. final def asInstanceOf[T0]: T0

    

    Definition Classes

    Any

13. def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

    

    Attributes

    protected

    Definition Classes

    AnnotatorModel

14. final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

    

    Attributes

    protected

    Definition Classes

    HasInputAnnotationCols

15. final def clear(param: Param[_]): NerOverwriter.this.type

    

    Definition Classes

    Params

16. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

17. def copy(extra: ParamMap): NerOverwriter

    

    requirement for annotators copies

    requirement for annotators copies

    Definition Classes

    RawAnnotator → Model → Transformer → PipelineStage → Params

18. def copyValues[T <: Params](to: T, extra: ParamMap): T

    

    Attributes

    protected

    Definition Classes

    Params

19. final def defaultCopy[T <: Params](extra: ParamMap): T

    

    Attributes

    protected

    Definition Classes

    Params

20. def dfAnnotate: UserDefinedFunction

    

    Wraps annotate to happen inside SparkSQL user defined functions in order to act with org.apache.spark.sql.Column

    Wraps annotate to happen inside SparkSQL user defined functions in order to act with org.apache.spark.sql.Column

    returns

    udf function to be applied to inputCols using this annotator's annotate function as part of ML transformation

    Definition Classes

    HasSimpleAnnotate

21. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

22. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

23. def explainParam(param: Param[_]): String

    

    Definition Classes

    Params

24. def explainParams(): String

    

    Definition Classes

    Params

25. def extraValidate(structType: StructType): Boolean

    

    Attributes

    protected

    Definition Classes

    RawAnnotator

26. def extraValidateMsg: String

    

    Override for additional custom schema checks

    Override for additional custom schema checks

    Attributes

    protected

    Definition Classes

    RawAnnotator

27. final def extractParamMap(): ParamMap

    

    Definition Classes

    Params

28. final def extractParamMap(extra: ParamMap): ParamMap

    

    Definition Classes

    Params

29. val features: ArrayBuffer[Feature[_, _, _]]

    

    Definition Classes

    HasFeatures

30. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

31. def get[T](feature: StructFeature[T]): Option[T]

    

    Attributes

    protected

    Definition Classes

    HasFeatures

32. def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

    

    Attributes

    protected

    Definition Classes

    HasFeatures

33. def get[T](feature: SetFeature[T]): Option[Set[T]]

    

    Attributes

    protected

    Definition Classes

    HasFeatures

34. def get[T](feature: ArrayFeature[T]): Option[Array[T]]

    

    Attributes

    protected

    Definition Classes

    HasFeatures

35. final def get[T](param: Param[T]): Option[T]

    

    Definition Classes

    Params

36. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

37. final def getDefault[T](param: Param[T]): Option[T]

    

    Definition Classes

    Params

38. def getInputCols: Array[String]

    
    returns

    input annotations columns currently used

    Definition Classes

    HasInputAnnotationCols

39. def getLazyAnnotator: Boolean

    

    Definition Classes

    CanBeLazy

40. def getNewResult: String

    

    New NER class to overwrite

41. final def getOrDefault[T](param: Param[T]): T

    

    Definition Classes

    Params

42. final def getOutputCol: String

    

    Gets annotation column name going to generate

    Gets annotation column name going to generate

    Definition Classes

    HasOutputAnnotationCol

43. def getParam(paramName: String): Param[Any]

    

    Definition Classes

    Params

44. def getStopWords: Array[String]

    

    the words to be filtered out.

45. final def hasDefault[T](param: Param[T]): Boolean

    

    Definition Classes

    Params

46. def hasParam(paramName: String): Boolean

    

    Definition Classes

    Params

47. def hasParent: Boolean

    

    Definition Classes

    Model

48. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

49. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

50. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

51. val inputAnnotatorTypes: Array[AnnotatorType]

    

    Input Annotator Type : NAMED_ENTITY

    Input Annotator Type : NAMED_ENTITY

    Definition Classes

    NerOverwriter → HasInputAnnotationCols

52. final val inputCols: StringArrayParam

    

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    Attributes

    protected

    Definition Classes

    HasInputAnnotationCols

53. final def isDefined(param: Param[_]): Boolean

    

    Definition Classes

    Params

54. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

55. final def isSet(param: Param[_]): Boolean

    

    Definition Classes

    Params

56. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

57. val lazyAnnotator: BooleanParam

    

    Definition Classes

    CanBeLazy

58. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

59. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

60. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

61. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

62. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

63. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

64. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

65. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

66. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

67. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

68. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

69. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

70. def msgHelper(schema: StructType): String

    

    Attributes

    protected

    Definition Classes

    HasInputAnnotationCols

71. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

72. val newResult: Param[String]

    

    New NER class to overwrite

73. final def notify(): Unit

    

    Definition Classes

    AnyRef

74. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

75. def onWrite(path: String, spark: SparkSession): Unit

    

    Attributes

    protected

    Definition Classes

    ParamsAndFeaturesWritable

76. val outputAnnotatorType: AnnotatorType

    

    Output Annotator Type : NAMED_ENTITY

    Output Annotator Type : NAMED_ENTITY

    Definition Classes

    NerOverwriter → HasOutputAnnotatorType

77. final val outputCol: Param[String]

    

    Attributes

    protected

    Definition Classes

    HasOutputAnnotationCol

78. lazy val params: Array[Param[_]]

    

    Definition Classes

    Params

79. var parent: Estimator[NerOverwriter]

    

    Definition Classes

    Model

80. def save(path: String): Unit

    

    Definition Classes

    MLWritable

    Annotations

    @Since( "1.6.0" ) @throws( ... )

81. def set[T](feature: StructFeature[T], value: T): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

82. def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

83. def set[T](feature: SetFeature[T], value: Set[T]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

84. def set[T](feature: ArrayFeature[T], value: Array[T]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

85. final def set(paramPair: ParamPair[_]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    Params

86. final def set(param: String, value: Any): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    Params

87. final def set[T](param: Param[T], value: T): NerOverwriter.this.type

    

    Definition Classes

    Params

88. def setDefault[T](feature: StructFeature[T], value: () ⇒ T): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

89. def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

90. def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

91. def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    HasFeatures

92. final def setDefault(paramPairs: ParamPair[_]*): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    Params

93. final def setDefault[T](param: Param[T], value: T): NerOverwriter.this.type

    

    Attributes

    protected

    Definition Classes

    Params

94. final def setInputCols(value: String*): NerOverwriter.this.type

    

    Definition Classes

    HasInputAnnotationCols

95. final def setInputCols(value: Array[String]): NerOverwriter.this.type

    

    Overrides required annotators column if different than default

    Overrides required annotators column if different than default

    Definition Classes

    HasInputAnnotationCols

96. def setLazyAnnotator(value: Boolean): NerOverwriter.this.type

    

    Definition Classes

    CanBeLazy

97. def setNewResult(r: String): NerOverwriter.this.type

    

    New NER class to overwrite

98. final def setOutputCol(value: String): NerOverwriter.this.type

    

    Overrides annotation column name when transforming

    Overrides annotation column name when transforming

    Definition Classes

    HasOutputAnnotationCol

99. def setParent(parent: Estimator[NerOverwriter]): NerOverwriter

    

    Definition Classes

    Model

100. def setStopWords(value: Array[String]): NerOverwriter.this.type

     

     the words to be filtered out.

101. val stopWords: StringArrayParam

     

     the words to be filtered out.

102. final def synchronized[T0](arg0: ⇒ T0): T0

     

     Definition Classes

     AnyRef

103. def toString(): String

     

     Definition Classes

     Identifiable → AnyRef → Any

104. final def transform(dataset: Dataset[_]): DataFrame

     

     Given requirements are met, this applies ML transformation within a Pipeline or stand-alone Output annotation will be generated as a new column, previous annotations are still available separately metadata is built at schema level to record annotations structural information outside its content

     Given requirements are met, this applies ML transformation within a Pipeline or stand-alone Output annotation will be generated as a new column, previous annotations are still available separately metadata is built at schema level to record annotations structural information outside its content

     dataset

     Dataset[Row]

     Definition Classes

     AnnotatorModel → Transformer

105. def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

     

     Definition Classes

     Transformer

     Annotations

     @Since( "2.0.0" )

106. def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

     

     Definition Classes

     Transformer

     Annotations

     @Since( "2.0.0" ) @varargs()

107. final def transformSchema(schema: StructType): StructType

     

     requirement for pipeline transformation validation.

     requirement for pipeline transformation validation. It is called on fit()

     Definition Classes

     RawAnnotator → PipelineStage

108. def transformSchema(schema: StructType, logging: Boolean): StructType

     

     Attributes

     protected

     Definition Classes

     PipelineStage

     Annotations

     @DeveloperApi()

109. val uid: String

     

     Definition Classes

     NerOverwriter → Identifiable

110. def validate(schema: StructType): Boolean

     

     takes a Dataset and checks to see if all the required annotation types are present.

     takes a Dataset and checks to see if all the required annotation types are present.

     schema

     to be validated

     returns

     True if all the required types are present, else false

     Attributes

     protected

     Definition Classes

     RawAnnotator

111. final def wait(): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

112. final def wait(arg0: Long, arg1: Int): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

113. final def wait(arg0: Long): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

114. def wrapColumnMetadata(col: Column): Column

     

     Attributes

     protected

     Definition Classes

     RawAnnotator

115. def write: MLWriter

     

     Definition Classes

     ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable