SchemaUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def appendFlattenedStructsToDataFrame(df: DataFrame, prefixForNewColumns: String): DataFrame

   

5. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

6. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

7. def collectMaxElementSizes(rdd: RDD[Row], schema: StructType): Array[Int]

   

   Collect max size of each element in DataFrame.

   Collect max size of each element in DataFrame. For array -> max array size For vectors -> max vector size For simple types -> 1

   returns

   array containing size of each element

8. def collectVectorLikeTypes(flatSchema: StructType): Seq[Int]

   

9. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

10. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

11. def expandedSchema(flatSchema: StructType, elemMaxSizes: Array[Int]): Seq[StructField]

    

    Returns expanded schema

    Returns expanded schema

    • schema is represented as list of types
    • all arrays are expanded into columns based on the longest one
    • all vectors are expanded into columns based on the longest one

    flatSchema

    flat schema of spark data frame

    returns

    list of types with their positions

12. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

13. def flattenDataFrame(df: DataFrame, flatSchema: StructType): DataFrame

    

14. def flattenDataFrame(df: DataFrame): DataFrame

    

15. def flattenSchema(df: DataFrame): StructType

    

16. def flattenStructsInDataFrame(df: DataFrame): DataFrame

    

17. def flattenStructsInSchema(schema: StructType, sourceColPrefix: Option[String] = None, targetColPrefix: Option[String] = None, nullable: Boolean = false): Seq[(StructField, String)]

    

18. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

19. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

22. final def notify(): Unit

    

    Definition Classes

    AnyRef

23. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

24. def rowsToRowSchemas(df: DataFrame): Dataset[ArrayBuffer[FieldWithOrder]]

    

25. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

26. def toString(): String

    

    Definition Classes

    AnyRef → Any

27. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

28. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

29. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )