SplitOperators

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def chronologicalSplit(data: DataFrame, seed: Long, trainSplitChronologicalColumn: String, trainSplitChronologicalRandomPercentage: Double, trainPortion: Double): Array[DataFrame]

   

6. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

7. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

8. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

9. def finalize(): Unit

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( classOf[java.lang.Throwable] )

10. def genTestTrain(data: DataFrame, seed: Long, uniqueLabels: Array[Row], trainSplitMethod: String, labelCol: String, trainPortion: Double, syntheticCol: String = "syntheticColumn", trainSplitChronologicalColumn: String = "datetime", trainSplitChronologicalRandomPercentage: Double = 0.05, reductionFactor: Double = 0.5): Array[DataFrame]

    

11. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

12. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

13. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

14. def kSamplingSplit(data: DataFrame, seed: Long, uniqueLabels: Array[Row], syntheticCol: String, labelCol: String, trainPortion: Double): Array[DataFrame]

    

    Split methodology for getting test and train of KSample up-sampled data.
    Both data sets are split into test and train.

    Split methodology for getting test and train of KSample up-sampled data.
    Both data sets are split into test and train.
    The returned collections are a union of the real train + synthetic train, but only the real test data.

    data

    DataFrame: The full data set (containing a synthetic column that indicates whether the data is real or not)

    seed

    Long: A seed value that is consistent across both data sets

    uniqueLabels

    Array[Row]: The unique entries of the label values

    returns

    Array[DataFrame] of Array(trainData, testData)

    Since

    0.5.1

15. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

16. final def notify(): Unit

    

    Definition Classes

    AnyRef

17. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

18. def optimizeTestTrain(train: DataFrame, test: DataFrame, optimalParts: Int, shuffle: Boolean = false): (DataFrame, DataFrame)

    

19. def overSampleSplit(data: DataFrame, seed: Long, labelCol: String, trainPortion: Double): Array[DataFrame]

    

20. lazy val sc: SparkContext

    

    Definition Classes

    SparkSessionWrapper

21. lazy val spark: SparkSession

    

    Definition Classes

    SparkSessionWrapper

22. def stratifiedSplit(data: DataFrame, seed: Long, uniqueLabels: Array[Row], labelCol: String, trainPortion: Double): Array[DataFrame]

    

    Method for stratification of the test/train around the unique values of the label column This mode is recommended for label value distributions in classification that have relatively balanced and uniformly distributed instances of the classes.

    Method for stratification of the test/train around the unique values of the label column This mode is recommended for label value distributions in classification that have relatively balanced and uniformly distributed instances of the classes. If there is significant skew, it is highly recommended to use under or over sampling.

    data

    Dataframe that is the input to the train/test split

    seed

    random seed for splitting the data into train/test.

    returns

    An Array of Dataframes: Array[<trainData>, <testData>]

23. def stratifyReduce(data: DataFrame, reductionFactor: Double, seed: Long, uniqueLabels: Array[Row], labelCol: String, trainPortion: Double): Array[DataFrame]

    

24. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

25. def toString(): String

    

    Definition Classes

    AnyRef → Any

26. def underSampleSplit(data: DataFrame, seed: Long, labelCol: String, trainPortion: Double): Array[DataFrame]

    

27. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

28. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

29. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )