UDFUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def call_udf(udfName: String, cols: Seq[Column]): Column
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate() @throws( ... )
def createLookup(name: String, df: DataFrame, spark: SparkSession, keyCols: List[String], rowCols: List[String]): UserDefinedFunction

Function registers 4 different UDFs with spark registry.
Function registers 4 different UDFs with spark registry. UDF for lookup_match, lookup_count, lookup_row and lookup functions are registered. This function stores the data of input dataframe in a broadcast variable, then uses this broadcast variable in different lookup functions.
lookup : This function returns the first matching row for given input keys lookup_count : This function returns the count of all matching rows for given input keys. lookup_match : This function returns 0 if there is no matching row and 1 for some matching rows for given input keys. lookup_row : This function returns all the matching rows for given input keys.
This function registers for upto 10 matching keys as input to these lookup functions.
name
UDF Name
df
input dataframe
spark
spark session
keyCols
columns to be used as keys in lookup functions.
rowCols
schema of entire row which will be stored for each matching key.
returns
registered UDF definitions for lookup functions. These UDF functions returns different results depending on the lookup function.
def createRangeLookup(name: String, df: DataFrame, spark: SparkSession, minColumn: String, maxColumn: String, valueColumns: List[String]): UserDefinedFunction

Method to create UDF which looks for passed input double in input dataframe.
Method to create UDF which looks for passed input double in input dataframe. This function first loads the data of dataframe in broadcast variable and then defines a UDF which looks for input double value in the data stored in broadcast variable. If input double lies between passed col1 and col2 values then it adds corresponding row in the returned result. If value of input double doesn't lie between col1 and col2 then it simply returns null for current row in result.
name
created UDF name
df
input dataframe
spark
spark session
minColumn
column whose value to be considered as minimum in comparison.
maxColumn
column whose value to be considered as maximum in comparison.
valueColumns
remaining column names to be part of result.
returns
registers UDF which in turn returns rows corresponding to each row in dataframe on which range UDF is called.
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def lookup(lookupName: String, cols: List[Column]): Column

By default returns only the first matching record
def lookup_count(lookupName: String, cols: List[Column]): Column
def lookup_last(lookupName: String, cols: List[Column]): Column

Returns the last matching record
def lookup_match(lookupName: String, cols: List[Column]): Column

returns
Boolean Column
def lookup_nth(lookupName: String, cols: List[Column]): Column
def lookup_row(lookupName: String, cols: List[Column]): Column
def lookup_row_reverse(lookupName: String, cols: List[Column]): Column
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@Deprecated @deprecated @throws( classOf[java.lang.Throwable] )
Deprecated
(Since version ) see corresponding Javadoc for more information.

Related Doc: package python

object UDFUtils

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def call_udf(udfName: String, cols: Seq[Column]): Column

def clone(): AnyRef

def createLookup(name: String, df: DataFrame, spark: SparkSession, keyCols: List[String], rowCols: List[String]): UserDefinedFunction

def createRangeLookup(name: String, df: DataFrame, spark: SparkSession, minColumn: String, maxColumn: String, valueColumns: List[String]): UserDefinedFunction

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def lookup(lookupName: String, cols: List[Column]): Column

def lookup_count(lookupName: String, cols: List[Column]): Column

def lookup_last(lookupName: String, cols: List[Column]): Column

def lookup_match(lookupName: String, cols: List[Column]): Column

def lookup_nth(lookupName: String, cols: List[Column]): Column

def lookup_row(lookupName: String, cols: List[Column]): Column

def lookup_row_reverse(lookupName: String, cols: List[Column]): Column

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def wait(): Unit

Deprecated Value Members

def finalize(): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped