CustomSparkAction

Instance Constructors

new CustomSparkAction(id: ActionObjectId, inputIds: Seq[DataObjectId], outputIds: Seq[DataObjectId], transformer: CustomDfsTransformerConfig, breakDataFrameLineage: Boolean = false, persist: Boolean = false, initExecutionMode: Option[ExecutionMode] = None, metadata: Option[ActionMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

inputIds
input DataObject's
outputIds
output DataObject's
transformer
Custom Transformer to transform Seq[DataFrames]

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addRuntimeEvent(phase: String, state: RuntimeEventState, msg: Option[String] = None, results: Seq[SubFeed] = Seq()): Unit

Adds an action event
Adds an action event

Definition Classes
Action
final def asInstanceOf[T0]: T0

Definition Classes
Any
val breakDataFrameLineage: Boolean

Stop propagating input DataFrame through action and instead get a new DataFrame from DataObject This is needed if the input DataFrame includes many transformations from previous Actions.
Stop propagating input DataFrame through action and instead get a new DataFrame from DataObject This is needed if the input DataFrame includes many transformations from previous Actions.

Definition Classes
CustomSparkAction → SparkSubFeedsAction
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def enableRuntimeMetrics(): Unit

Runtime metrics
Runtime metrics
Note: runtime metrics are disabled by default, because they are only collected when running Actions from an ActionDAG. This is not the case for Tests other use cases. If enabled exceptions are thrown if metrics are not found.

Definition Classes
Action
def enrichSubFeedsDataFrame(inputs: Seq[DataObject with CanCreateDataFrame], subFeeds: Seq[SparkSubFeed])(implicit session: SparkSession): Seq[SparkSubFeed]

Enriches SparkSubFeeds with DataFrame if not existing
Enriches SparkSubFeeds with DataFrame if not existing
inputs
input data objects.
subFeeds
input SubFeeds.

Attributes
protected
Definition Classes
SparkSubFeedsAction
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def exec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Action.exec implementation
Action.exec implementation
subFeeds
SparkSubFeed's to be processed
returns
processed SparkSubFeed's

Definition Classes
SparkSubFeedsAction → Action
def factory: FromConfigFactory[Action]

Returns the factory that can parse this type (that is, type CO).
Returns the factory that can parse this type (that is, type CO).
Typically, implementations of this method should return the companion object of the implementing class. The companion object in turn should implement FromConfigFactory.
returns
the factory (object) for this class.

Definition Classes
CustomSparkAction → ParsableFromConfig
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getAllLatestMetrics: Map[DataObjectId, Option[ActionMetrics]]

Definition Classes
Action
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getFinalMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

Definition Classes
Action
def getInputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

Attributes
protected
Definition Classes
Action
def getLatestMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

Definition Classes
Action
def getOutputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

Attributes
protected
Definition Classes
Action
def getRuntimeInfo: Option[RuntimeInfo]

get latest runtime information for this action
get latest runtime information for this action

Definition Classes
Action
val id: ActionObjectId

A unique identifier for this instance.
A unique identifier for this instance.

Definition Classes
CustomSparkAction → Action → SdlConfigObject
final def init(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Generic init implementation for Action.init
Generic init implementation for Action.init
subFeeds
SparkSubFeed's to be processed
returns
processed SparkSubFeed's

Definition Classes
SparkSubFeedsAction → Action
val initExecutionMode: Option[ExecutionMode]

Execution mode if this Action is a start node of a DAG run
Execution mode if this Action is a start node of a DAG run

Definition Classes
CustomSparkAction → SparkSubFeedsAction
val inputIds: Seq[DataObjectId]

input DataObject's
val inputs: Seq[DataObject with CanCreateDataFrame]

Input DataObjects To be implemented by subclasses
Input DataObjects To be implemented by subclasses

Definition Classes
CustomSparkAction → SparkSubFeedsAction → Action
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
lazy val mainInput: Option[DataObject with CanCreateDataFrame]

Attributes
protected
Definition Classes
SparkSubFeedsAction
lazy val mainOutput: Option[DataObject with CanWriteDataFrame]

Attributes
protected
Definition Classes
SparkSubFeedsAction
val metadata: Option[ActionMetadata]

Additional metadata for the Action
Additional metadata for the Action

Definition Classes
CustomSparkAction → Action
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def nodeId: String

provide an implementation of the DAG node id
provide an implementation of the DAG node id

Definition Classes
Action → DAGNode
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def onRuntimeMetrics(dataObjectId: Option[DataObjectId], metrics: ActionMetrics): Unit

Definition Classes
Action
val outputIds: Seq[DataObjectId]

output DataObject's
val outputs: Seq[DataObject with CanWriteDataFrame]

Output DataObjects To be implemented by subclasses
Output DataObjects To be implemented by subclasses

Definition Classes
CustomSparkAction → SparkSubFeedsAction → Action
val persist: Boolean

Force persisting DataFrame on Disk.
Force persisting DataFrame on Disk. This helps to reduce memory needed for caching the DataFrame content and can serve as a recovery point in case an task get's lost.

Definition Classes
CustomSparkAction → SparkSubFeedsAction
def postExec(inputSubFeed: Seq[SubFeed], outputSubFeed: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed after executing an action.
Executes operations needed after executing an action. In this step any operation on Input- or Output-DataObjects needed after the main task is executed, e.g. JdbcTableDataObjects postSql or CopyActions deleteInputData.

Definition Classes
Action
def preExec(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed before executing an action.
Executes operations needed before executing an action. In this step any operation on Input- or Output-DataObjects needed before the main task is executed, e.g. JdbcTableDataObjects preSql

Definition Classes
Action
def prepare(implicit session: SparkSession, context: ActionPipelineContext): Unit

Prepare DataObjects prerequisites.
Prepare DataObjects prerequisites. In this step preconditions are prepared & tested: - directories exists or can be created - connections can be created
This runs during the "prepare" operation of the DAG.

Definition Classes
Action
def setSparkJobMetadata(operation: Option[String] = None)(implicit session: SparkSession): Unit

Sets the util job description for better traceability in the Spark UI
Sets the util job description for better traceability in the Spark UI
Note: This sets Spark local properties, which are propagated to the respective executor tasks. We rely on this to match metrics back to Actions and DataObjects. As writing to a DataObject on the Driver happens uninterrupted in the same exclusive thread, this is suitable.
operation
operation description (be short...)

Definition Classes
Action
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final def toString(): String

This is displayed in ascii graph visualization
This is displayed in ascii graph visualization

Definition Classes
Action → AnyRef → Any
def toStringMedium: String

Definition Classes
Action
def toStringShort: String

Definition Classes
Action
def transform(subFeeds: Seq[SparkSubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SparkSubFeed]

Transform SparkSubFeed's.
Transform SparkSubFeed's. To be implemented by subclasses.
subFeeds
SparkSubFeed's to be transformed
returns
transformed SparkSubFeed's

Definition Classes
CustomSparkAction → SparkSubFeedsAction
val transformer: CustomDfsTransformerConfig

Custom Transformer to transform Seq[DataFrames]
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object CustomSparkAction | package action

Instance Constructors

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addRuntimeEvent(phase: String, state: RuntimeEventState, msg: Option[String] = None, results: Seq[SubFeed] = Seq()): Unit

final def asInstanceOf[T0]: T0

val breakDataFrameLineage: Boolean

def clone(): AnyRef

def enableRuntimeMetrics(): Unit

def enrichSubFeedsDataFrame(inputs: Seq[DataObject with CanCreateDataFrame], subFeeds: Seq[SparkSubFeed])(implicit session: SparkSession): Seq[SparkSubFeed]

final def eq(arg0: AnyRef): Boolean

final def exec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

def factory: FromConfigFactory[Action]

def finalize(): Unit

def getAllLatestMetrics: Map[DataObjectId, Option[ActionMetrics]]

final def getClass(): Class[_]

def getFinalMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

def getInputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getLatestMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

def getOutputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getRuntimeInfo: Option[RuntimeInfo]

val id: ActionObjectId

final def init(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

val initExecutionMode: Option[ExecutionMode]

val inputIds: Seq[DataObjectId]

val inputs: Seq[DataObject with CanCreateDataFrame]

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

lazy val mainInput: Option[DataObject with CanCreateDataFrame]

lazy val mainOutput: Option[DataObject with CanWriteDataFrame]

val metadata: Option[ActionMetadata]

final def ne(arg0: AnyRef): Boolean

def nodeId: String

final def notify(): Unit

final def notifyAll(): Unit

def onRuntimeMetrics(dataObjectId: Option[DataObjectId], metrics: ActionMetrics): Unit

val outputIds: Seq[DataObjectId]

val outputs: Seq[DataObject with CanWriteDataFrame]

val persist: Boolean

def postExec(inputSubFeed: Seq[SubFeed], outputSubFeed: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

def preExec(implicit session: SparkSession, context: ActionPipelineContext): Unit

def prepare(implicit session: SparkSession, context: ActionPipelineContext): Unit

def setSparkJobMetadata(operation: Option[String] = None)(implicit session: SparkSession): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

final def toString(): String

def toStringMedium: String

def toStringShort: String

def transform(subFeeds: Seq[SparkSubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SparkSubFeed]

val transformer: CustomDfsTransformerConfig

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from SparkSubFeedsAction

Inherited from Action

Inherited from SmartDataLakeLogger

Inherited from DAGNode

Inherited from ParsableFromConfig[Action]

Inherited from SdlConfigObject

Inherited from AnyRef

Inherited from Any

Ungrouped