CustomDfCreatorConfig

Configuration of a custom Spark-DataFrame creator as part of CustomDfDataObject Define a exec function which receives a map of options and returns a DataFrame to be used as input. Optionally define a schema function to return a StructType used as schema in init-phase. See also trait CustomDfCreator.

Note that for now implementing CustomDfCreator.schema method is only possible with className configuration attribute.

className: Optional class name implementing trait CustomDfCreator
scalaFile: Optional file where scala code for creator is loaded from. The scala code in the file needs to be a function of type fnExecType.
scalaCode: Optional scala code for creator. The scala code needs to be a function of type fnExecType.
options: Options to pass to the creator

Linear Supertypes

Serializable, Serializable, Product, Equals, AnyRef, Any

Instance Constructors

new CustomDfCreatorConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, options: Option[Map[String, String]] = None)

className
Optional class name implementing trait CustomDfCreator
scalaFile
Optional file where scala code for creator is loaded from. The scala code in the file needs to be a function of type fnExecType.
scalaCode
Optional scala code for creator. The scala code needs to be a function of type fnExecType.
options
Options to pass to the creator

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
val className: Option[String]

Optional class name implementing trait CustomDfCreator
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def exec(implicit session: SparkSession): DataFrame
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
val fnEmptySchema: fnSchemaType
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
val impl: CustomDfCreator
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val options: Option[Map[String, String]]

Options to pass to the creator
val scalaCode: Option[String]

Optional scala code for creator.
Optional scala code for creator. The scala code needs to be a function of type fnExecType.
val scalaFile: Option[String]

Optional file where scala code for creator is loaded from.
Optional file where scala code for creator is loaded from. The scala code in the file needs to be a function of type fnExecType.
def schema(implicit session: SparkSession): Option[StructType]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
CustomDfCreatorConfig → AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object CustomDfCreatorConfig | package customlogic

case class CustomDfCreatorConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, options: Option[Map[String, String]] = None) extends Product with Serializable

Instance Constructors

new CustomDfCreatorConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, options: Option[Map[String, String]] = None)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

val className: Option[String]

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def exec(implicit session: SparkSession): DataFrame

def finalize(): Unit

val fnEmptySchema: fnSchemaType

final def getClass(): Class[_]

val impl: CustomDfCreator

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val options: Option[Map[String, String]]

val scalaCode: Option[String]

val scalaFile: Option[String]

def schema(implicit session: SparkSession): Option[StructType]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped