DsvIngestionUtil

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

7. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

8. def finalize(): Unit

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( classOf[java.lang.Throwable] )

9. final def getClass(): Class[_]

   

   Definition Classes

   AnyRef → Any

10. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

11. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

12. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

13. final def notify(): Unit

    

    Definition Classes

    AnyRef

14. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

15. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

16. def toString(): String

    

    Definition Classes

    AnyRef → Any

17. def validate(session: SparkSession, dataset: DataFrame, attributes: List[Attribute], types: List[Type], sparkType: StructType)(implicit settings: Settings): (RDD[String], RDD[Row])

    

    For each col of each row

    For each col of each row

    • we extract the col value / the col constraints / col type
    • we check that the constraints are verified
    • we apply any required privacy transformation
    • parse the column into the target primitive Spark Type We end up using catalyst to create a Spark Row

    session

    : The Spark session

    dataset

    : The dataset

    attributes

    : the col attributes

    types

    : List of globally defined types

    sparkType

    : The expected Spark Type for valid rows

    returns

    Two RDDs : One RDD for rejected rows and one RDD for accepted rows

18. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

19. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

20. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )