SparkSubFeedsAction

abstract def breakDataFrameLineage: Boolean

Stop propagating input DataFrame through action and instead get a new DataFrame from DataObject.

abstract def executionMode: Option[ExecutionMode]

execution mode for this action.

abstract def factory: FromConfigFactory[Action]

Returns the factory that can parse this type (that is, type CO).

abstract val id: ActionObjectId

A unique identifier for this instance.

abstract def inputs: Seq[DataObject with CanCreateDataFrame]

Input DataObjects To be implemented by subclasses

abstract def mainInputId: Option[DataObjectId]

abstract def mainOutputId: Option[DataObjectId]

abstract def metadata: Option[ActionMetadata]

Additional metadata for the Action

abstract def metricsFailCondition: Option[String]

Spark SQL condition evaluated as where-clause against dataframe of metrics.

abstract def outputs: Seq[DataObject with CanWriteDataFrame]

Output DataObjects To be implemented by subclasses

abstract def persist: Boolean

Force persisting DataFrame on Disk.

abstract def transform(subFeeds: Seq[SparkSubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SparkSubFeed]

Transform SparkSubFeed's.

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addRuntimeEvent(phase: ExecutionPhase, state: RuntimeEventState, msg: Option[String] = None, results: Seq[SubFeed] = Seq()): Unit

Adds an action event

def applyAdditionalColumns(additionalColumns: Map[String, String], partitionValues: Seq[PartitionValues])(df: DataFrame)(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

applies additionalColumns

def applyCastDecimal2IntegralFloat(df: DataFrame): DataFrame

applies type casting decimal -> integral/float

def applyCustomTransformation(transformer: CustomDfTransformerConfig, dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues])(df: DataFrame)(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

apply custom transformation

def applyFilter(filterClauseExpr: Column)(df: DataFrame): DataFrame

applies filterClauseExpr

def applyTransformations(inputSubFeed: SparkSubFeed, transformation: Option[CustomDfTransformerConfig], columnBlacklist: Option[Seq[String]], columnWhitelist: Option[Seq[String]], additionalColumns: Option[Map[String, String]], standardizeDatatypes: Boolean, additionalTransformers: Seq[(DataFrame) ⇒ DataFrame], filterClauseExpr: Option[Column] = None)(implicit session: SparkSession, context: ActionPipelineContext): SparkSubFeed

applies all the transformations above

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def enableRuntimeMetrics(): Unit

Runtime metrics

def enrichSubFeedDataFrame(input: DataObject with CanCreateDataFrame, subFeed: SparkSubFeed, phase: ExecutionPhase)(implicit session: SparkSession, context: ActionPipelineContext): SparkSubFeed

Enriches SparkSubFeed with DataFrame if not existing

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

final def exec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Action.exec implementation

def filterDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues], genericFilter: Option[Column]): DataFrame

Filter DataFrame with given partition values

def finalize(): Unit

def getAllLatestMetrics: Map[DataObjectId, Option[ActionMetrics]]

final def getClass(): Class[_]

def getFinalMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

def getInputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getLatestMetrics(dataObjectId: DataObjectId): Option[ActionMetrics]

def getLatestRuntimeState: Option[RuntimeEventState]

get latest runtime state

def getOutputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getRuntimeInfo: Option[RuntimeInfo]

get latest runtime information for this action

def hashCode(): Int

final def init(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Generic init implementation for Action.init

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

lazy val mainInput: DataObject with CanCreateDataFrame

lazy val mainOutput: DataObject with CanWriteDataFrame

def multiTransformSubfeed(subFeed: SparkSubFeed, transformers: Seq[(DataFrame) ⇒ DataFrame]): SparkSubFeed

applies multiple transformations to a sequence of subfeeds

final def ne(arg0: AnyRef): Boolean

def nodeId: String

provide an implementation of the DAG node id

final def notify(): Unit

final def notifyAll(): Unit

def onRuntimeMetrics(dataObjectId: Option[DataObjectId], metrics: ActionMetrics): Unit

def postExec(inputSubFeed: Seq[SubFeed], outputSubFeed: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed after executing an action.

def preExec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed before executing an action.

def prepare(implicit session: SparkSession, context: ActionPipelineContext): Unit

Prepare DataObjects prerequisites.

def prepareInputSubFeed(subFeed: SparkSubFeed, input: DataObject with CanCreateDataFrame)(implicit session: SparkSession, context: ActionPipelineContext): SparkSubFeed

Applies changes to a SubFeed from a previous action in order to be used as input for this actions transformation.

def recursiveInputs: Seq[DataObject with CanCreateDataFrame]

Recursive Inputs are DataObjects that are used as Output and Input in the same action This is usually prohibited as it creates loops in the DAG.

def reset(): Unit

Resets the runtime state of this Action This is mainly used for testing

def setSparkJobMetadata(operation: Option[String] = None)(implicit session: SparkSession): Unit

Sets the util job description for better traceability in the Spark UI

final def synchronized[T0](arg0: ⇒ T0): T0

final def toString(): String

This is displayed in ascii graph visualization

def toStringMedium: String

def toStringShort: String

def updateSubFeedAfterWrite(subFeed: SparkSubFeed)(implicit session: SparkSession, context: ActionPipelineContext): SparkSubFeed

def validateAndUpdateSubFeedPartitionValues(output: DataObject, subFeed: SparkSubFeed)(implicit session: SparkSession): SparkSubFeed

Updates the partition values of a SubFeed to the partition columns of an output, removing not existing columns from the partition values.

def validateDataFrameContainsCols(df: DataFrame, columns: Seq[String], debugName: String): Unit

Validate that DataFrame contains a given list of columns, throwing an exception otherwise.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def writeSubFeed(subFeed: SparkSubFeed, output: DataObject with CanWriteDataFrame, isRecursiveInput: Boolean = false)(implicit session: SparkSession): Boolean

writes subfeed to output respecting given execution mode