CustomDfsTransformerConfig

Configuration of a custom Spark-DataFrame transformation between several inputs and outputs (n:m)

className: Optional class name to load transformer code from
scalaFile: Optional file where scala code for transformation is loaded from
scalaCode: Optional scala code for transformation
sqlCode: Optional map of DataObjectId and corresponding SQL Code. Use tokens %{<key>} to replace with runtimeOptions in SQL code. Example: "select * from test where run = %{runId}"
options: Options to pass to the transformation
runtimeOptions: optional tuples of [key, spark sql expression] to be added as additional options when executing transformation. The spark sql expressions are evaluated against an instance of DefaultExpressionData.

Linear Supertypes

Serializable, Serializable, Product, Equals, AnyRef, Any

Instance Constructors

new CustomDfsTransformerConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, sqlCode: Map[DataObjectId, String] = Map(), options: Map[String, String] = Map(), runtimeOptions: Map[String, String] = Map())

className
Optional class name to load transformer code from
scalaFile
Optional file where scala code for transformation is loaded from
scalaCode
Optional scala code for transformation
sqlCode
Optional map of DataObjectId and corresponding SQL Code. Use tokens %{<key>} to replace with runtimeOptions in SQL code. Example: "select * from test where run = %{runId}"
options
Options to pass to the transformation
runtimeOptions
optional tuples of [key, spark sql expression] to be added as additional options when executing transformation. The spark sql expressions are evaluated against an instance of DefaultExpressionData.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
val className: Option[String]

Optional class name to load transformer code from
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
val impl: Option[CustomDfsTransformer]
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val options: Map[String, String]

Options to pass to the transformation
val runtimeOptions: Map[String, String]

optional tuples of [key, spark sql expression] to be added as additional options when executing transformation.
optional tuples of [key, spark sql expression] to be added as additional options when executing transformation. The spark sql expressions are evaluated against an instance of DefaultExpressionData.
val scalaCode: Option[String]

Optional scala code for transformation
val scalaFile: Option[String]

Optional file where scala code for transformation is loaded from
val sqlCode: Map[DataObjectId, String]

Optional map of DataObjectId and corresponding SQL Code.
Optional map of DataObjectId and corresponding SQL Code. Use tokens %{<key>} to replace with runtimeOptions in SQL code. Example: "select * from test where run = %{runId}"
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
CustomDfsTransformerConfig → AnyRef → Any
def transform(actionId: ActionObjectId, partitionValues: Seq[PartitionValues], dfs: Map[String, DataFrame])(implicit session: SparkSession, context: ActionPipelineContext): Map[String, DataFrame]
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package customlogic

case class CustomDfsTransformerConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, sqlCode: Map[DataObjectId, String] = Map(), options: Map[String, String] = Map(), runtimeOptions: Map[String, String] = Map()) extends Product with Serializable

Instance Constructors

new CustomDfsTransformerConfig(className: Option[String] = None, scalaFile: Option[String] = None, scalaCode: Option[String] = None, sqlCode: Map[DataObjectId, String] = Map(), options: Map[String, String] = Map(), runtimeOptions: Map[String, String] = Map())

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

val className: Option[String]

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def finalize(): Unit

final def getClass(): Class[_]

val impl: Option[CustomDfsTransformer]

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val options: Map[String, String]

val runtimeOptions: Map[String, String]

val scalaCode: Option[String]

val scalaFile: Option[String]

val sqlCode: Map[DataObjectId, String]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def transform(actionId: ActionObjectId, partitionValues: Seq[PartitionValues], dfs: Map[String, DataFrame])(implicit session: SparkSession, context: ActionPipelineContext): Map[String, DataFrame]

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped