SchemaUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def alignSchema(df: DataFrame, selector: List[Column]): DataFrame

Using schema selector returned from getDataFrameSelector aligns the schema of a DataFrame to the selector for operations where schema order might be important (e.g.
Using schema selector returned from getDataFrameSelector aligns the schema of a DataFrame to the selector for operations where schema order might be important (e.g. hashing the whole rows and using except)
df
DataFrame to have it's schema aligned/sorted
selector
model structType for the alignment of df
returns
Returns aligned and filtered schema
def alignSchema(df: DataFrame, structType: StructType): DataFrame

Using schema selector from getDataFrameSelector aligns the schema of a DataFrame to the selector for operations where schema order might be important (e.g.
Using schema selector from getDataFrameSelector aligns the schema of a DataFrame to the selector for operations where schema order might be important (e.g. hashing the whole rows and using except)
df
DataFrame to have it's schema aligned/sorted
structType
model structType for the alignment of df
returns
Returns aligned and filtered schema
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate() @throws( ... )
def diffSchema(schema1: StructType, schema2: StructType, parent: String = ""): Seq[String]

Returns a list of differences in one schema to the other
Returns a list of differences in one schema to the other
schema1
The first schema to compare
schema2
The second schema to compare
parent
Parent path. Should be left default by the users first run. This is used for the accumulation of differences and their print out.
returns
Returns a Seq of paths to differences in schemas
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def equivalentSchemas(schema1: StructType, schema2: StructType): Boolean

Compares 2 dataframe schemas.
Compares 2 dataframe schemas.
schema1
The first schema to compare
schema2
The second schema to compare
returns
true if provided schemas are the same ignoring nullability
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
def getDataFrameSelector(schema: StructType): List[Column]

Returns data selector that can be used to align schema of a data frame.
Returns data selector that can be used to align schema of a data frame. You can use alignSchema.
schema
Schema that serves as the model of column order
returns
Sorted DF to conform to schema
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isSubset(subsetSchema: StructType, originalSchema: StructType): Boolean

Checks if the originalSchema is a subset of subsetSchema.
Checks if the originalSchema is a subset of subsetSchema.
subsetSchema
The schema that needs to be extracted
originalSchema
The schema that needs to have at least all t
returns
true if provided schemas are the same ignoring nullability
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@Deprecated @deprecated @throws( classOf[java.lang.Throwable] )
Deprecated
(Since version ) see corresponding Javadoc for more information.

Related Doc: package spark

object SchemaUtils

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def alignSchema(df: DataFrame, selector: List[Column]): DataFrame

def alignSchema(df: DataFrame, structType: StructType): DataFrame

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def diffSchema(schema1: StructType, schema2: StructType, parent: String = ""): Seq[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def equivalentSchemas(schema1: StructType, schema2: StructType): Boolean

final def getClass(): Class[_]

def getDataFrameSelector(schema: StructType): List[Column]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def isSubset(subsetSchema: StructType, originalSchema: StructType): Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def wait(): Unit

Deprecated Value Members

def finalize(): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped