PearsonFiltering

Instance Constructors

new PearsonFiltering(df: DataFrame, featureColumnListing: Array[String], modelType: String)

df
: DataFrame -> Dataset with a vectorized field of features, the feature columns, and a label column.
featureColumnListing
: Array[String] -> List of all fields that make up the feature vector Usage: val autoFiltered = new PearsonFiltering(featurizedData, fields) .setLabelCol("label") .setFeaturesCol("features") .setFilterStatistic("pearsonStat") .setFilterDirection("greater") .setFilterMode("auto") .setAutoFilterNTile(0.5) .filterFields

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def _allowableCardinalilties: List[String]

Definition Classes
DataValidation
def _allowableCategoricalFilterModes: List[String]

Definition Classes
DataValidation
def _allowableDateTimeConversions: List[String]

Definition Classes
DataValidation
final val _allowedFilterDirections: Array[String]

Definition Classes
SanitizerDefaults
final val _allowedFilterModes: Array[String]

Definition Classes
SanitizerDefaults
final val _allowedStats: Array[String]

Pearson Defaults
Pearson Defaults

Definition Classes
SanitizerDefaults
final val allowableScalers: Array[String]

Scaler Defaults
Scaler Defaults

Definition Classes
SanitizerDefaults
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def convertDateAndTime(df: DataFrame, dateFields: List[String], timeFields: List[String], mode: String): (DataFrame, List[String])

Definition Classes
DataValidation
def defaultFeaturesCol: String

Definition Classes
SanitizerDefaults
def defaultLabelCol: String

Global Defaults
Global Defaults

Definition Classes
SanitizerDefaults
def defaultPNorm: Double

Definition Classes
SanitizerDefaults
def defaultPearsonAutoFilterNTile: Double

Definition Classes
SanitizerDefaults
def defaultPearsonFilterDirection: String

Definition Classes
SanitizerDefaults
def defaultPearsonFilterManualValue: Double

Definition Classes
SanitizerDefaults
def defaultPearsonFilterMode: String

Definition Classes
SanitizerDefaults
def defaultPearsonFilterStatistic: String

Definition Classes
SanitizerDefaults
def defaultRenamedFeaturesCol: String

Definition Classes
SanitizerDefaults
def defaultScalerMax: Double

Definition Classes
SanitizerDefaults
def defaultScalerMin: Double

Definition Classes
SanitizerDefaults
def defaultScalerType: String

Definition Classes
SanitizerDefaults
def defaultStandardScalerMeanFlag: Boolean

Definition Classes
SanitizerDefaults
def defaultStandardScalerStdDevFlag: Boolean

Definition Classes
SanitizerDefaults
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def filterFields(ignoreFields: Array[String] = Array.empty[String]): DataFrame

Main entry point for Pearson Filtering
Main entry point for Pearson Filtering
ignoreFields
Fields that will be ignored from running a Pearson filter against.
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def generateAssembly(numericColumns: List[String], characterColumns: List[String], featureCol: String): (Array[StringIndexer], Array[String], VectorAssembler)

Definition Classes
DataValidation
def getAutoFilterNTile: Double
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getFeaturesCol: String
def getFilterDirection: String
def getFilterManualValue: Double
def getFilterMode: String
def getFilterStatistic: String
def getLabelCol: String
def getParallelism: Int
def hashCode(): Int

Definition Classes
AnyRef → Any
def indexStrings(categoricalFields: List[String]): (Array[StringIndexer], Array[String])

Definition Classes
DataValidation
def invalidateSelection(value: String, allowances: Seq[String]): String

Definition Classes
DataValidation
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def oneHotEncodeStrings(stringIndexedFields: List[String]): (OneHotEncoderEstimator, Array[String])

Definition Classes
DataValidation
def setAutoFilterNTile(value: Double): PearsonFiltering.this.type
def setFeaturesCol(value: String): PearsonFiltering.this.type
def setFilterDirection(value: String): PearsonFiltering.this.type
def setFilterManualValue(value: Int): PearsonFiltering.this.type
def setFilterManualValue(value: Double): PearsonFiltering.this.type
def setFilterMode(value: String): PearsonFiltering.this.type
def setFilterStatistic(value: String): PearsonFiltering.this.type
def setLabelCol(value: String): PearsonFiltering.this.type
def setParallelism(value: Int): PearsonFiltering.this.type
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def validateCardinality(df: DataFrame, stringFields: List[String], cardinalityLimit: Int = 500, parallelism: Int = 20): ValidatedCategoricalFields

Definition Classes
DataValidation
def validateFieldPresence(df: DataFrame, column: String): Unit

Definition Classes
DataValidation
def validateInputDataframe(df: DataFrame): Unit

Definition Classes
DataValidation
def validateLabelAndFeatures(df: DataFrame, labelCol: String, featureCol: String): Unit

Definition Classes
DataValidation
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package sanitize

class PearsonFiltering extends DataValidation with SanitizerDefaults

Instance Constructors

new PearsonFiltering(df: DataFrame, featureColumnListing: Array[String], modelType: String)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def _allowableCardinalilties: List[String]

def _allowableCategoricalFilterModes: List[String]

def _allowableDateTimeConversions: List[String]

final val _allowedFilterDirections: Array[String]

final val _allowedFilterModes: Array[String]

final val _allowedStats: Array[String]

final val allowableScalers: Array[String]

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def convertDateAndTime(df: DataFrame, dateFields: List[String], timeFields: List[String], mode: String): (DataFrame, List[String])

def defaultFeaturesCol: String

def defaultLabelCol: String

def defaultPNorm: Double

def defaultPearsonAutoFilterNTile: Double

def defaultPearsonFilterDirection: String

def defaultPearsonFilterManualValue: Double

def defaultPearsonFilterMode: String

def defaultPearsonFilterStatistic: String

def defaultRenamedFeaturesCol: String

def defaultScalerMax: Double

def defaultScalerMin: Double

def defaultScalerType: String

def defaultStandardScalerMeanFlag: Boolean

def defaultStandardScalerStdDevFlag: Boolean

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def filterFields(ignoreFields: Array[String] = Array.empty[String]): DataFrame

def finalize(): Unit

def generateAssembly(numericColumns: List[String], characterColumns: List[String], featureCol: String): (Array[StringIndexer], Array[String], VectorAssembler)

def getAutoFilterNTile: Double

final def getClass(): Class[_]

def getFeaturesCol: String

def getFilterDirection: String

def getFilterManualValue: Double

def getFilterMode: String

def getFilterStatistic: String

def getLabelCol: String

def getParallelism: Int

def hashCode(): Int

def indexStrings(categoricalFields: List[String]): (Array[StringIndexer], Array[String])

def invalidateSelection(value: String, allowances: Seq[String]): String

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def oneHotEncodeStrings(stringIndexedFields: List[String]): (OneHotEncoderEstimator, Array[String])

def setAutoFilterNTile(value: Double): PearsonFiltering.this.type

def setFeaturesCol(value: String): PearsonFiltering.this.type

def setFilterDirection(value: String): PearsonFiltering.this.type

def setFilterManualValue(value: Int): PearsonFiltering.this.type

def setFilterManualValue(value: Double): PearsonFiltering.this.type

def setFilterMode(value: String): PearsonFiltering.this.type

def setFilterStatistic(value: String): PearsonFiltering.this.type

def setLabelCol(value: String): PearsonFiltering.this.type

def setParallelism(value: Int): PearsonFiltering.this.type

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def validateCardinality(df: DataFrame, stringFields: List[String], cardinalityLimit: Int = 500, parallelism: Int = 20): ValidatedCategoricalFields

def validateFieldPresence(df: DataFrame, column: String): Unit

def validateInputDataframe(df: DataFrame): Unit

def validateLabelAndFeatures(df: DataFrame, labelCol: String, featureCol: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SanitizerDefaults

Inherited from DataValidation

Inherited from AnyRef

Inherited from Any

Ungrouped