SparkIncrementalMode

Compares max entry in "compare column" between mainOutput and mainInput and incrementally loads the delta. This mode works only with SparkSubFeeds. The filter is not propagated to following actions.

compareCol: a comparable column name existing in mainInput and mainOutput used to identify the delta. Column content should be bigger for newer records.
alternativeOutputId: optional alternative outputId of DataObject later in the DAG. This replaces the mainOutputId. It can be used to ensure processing all partitions over multiple actions in case of errors.

Linear Supertypes

Serializable, Serializable, Product, Equals, ExecutionModeWithMainInputOutput, ExecutionMode, SmartDataLakeLogger, AnyRef, Any

Instance Constructors

new SparkIncrementalMode(compareCol: String, alternativeOutputId: Option[DataObjectId] = None)

compareCol
a comparable column name existing in mainInput and mainOutput used to identify the delta. Column content should be bigger for newer records.
alternativeOutputId
optional alternative outputId of DataObject later in the DAG. This replaces the mainOutputId. It can be used to ensure processing all partitions over multiple actions in case of errors.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def alternativeOutput(implicit context: ActionPipelineContext): Option[DataObject]

Definition Classes
ExecutionModeWithMainInputOutput
val alternativeOutputId: Option[DataObjectId]

optional alternative outputId of DataObject later in the DAG.
optional alternative outputId of DataObject later in the DAG. This replaces the mainOutputId. It can be used to ensure processing all partitions over multiple actions in case of errors.

Definition Classes
SparkIncrementalMode → ExecutionModeWithMainInputOutput
def apply(actionId: ActionObjectId, mainInput: DataObject, mainOutput: DataObject, subFeed: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): Option[(Seq[PartitionValues], Option[String])]

Definition Classes
SparkIncrementalMode → ExecutionMode
def applyCondition: Option[String]

Definition Classes
ExecutionMode
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val compareCol: String

a comparable column name existing in mainInput and mainOutput used to identify the delta.
a comparable column name existing in mainInput and mainOutput used to identify the delta. Column content should be bigger for newer records.
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def evaluateApplyCondition(actionId: ActionObjectId, subFeed: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): Option[Boolean]

Definition Classes
ExecutionMode
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
def mainInputOutputNeeded: Boolean

Definition Classes
SparkIncrementalMode → ExecutionMode
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def prepare(actionId: ActionObjectId)(implicit session: SparkSession, context: ActionPipelineContext): Unit

Definition Classes
SparkIncrementalMode → ExecutionMode
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object SparkIncrementalMode | package definitions

case class SparkIncrementalMode(compareCol: String, alternativeOutputId: Option[DataObjectId] = None) extends ExecutionMode with ExecutionModeWithMainInputOutput with Product with Serializable

Instance Constructors

new SparkIncrementalMode(compareCol: String, alternativeOutputId: Option[DataObjectId] = None)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def alternativeOutput(implicit context: ActionPipelineContext): Option[DataObject]

val alternativeOutputId: Option[DataObjectId]

def apply(actionId: ActionObjectId, mainInput: DataObject, mainOutput: DataObject, subFeed: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): Option[(Seq[PartitionValues], Option[String])]

def applyCondition: Option[String]

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val compareCol: String

final def eq(arg0: AnyRef): Boolean

final def evaluateApplyCondition(actionId: ActionObjectId, subFeed: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): Option[Boolean]

def finalize(): Unit

final def getClass(): Class[_]

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

def mainInputOutputNeeded: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def prepare(actionId: ActionObjectId)(implicit session: SparkSession, context: ActionPipelineContext): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from ExecutionModeWithMainInputOutput

Inherited from ExecutionMode

Inherited from SmartDataLakeLogger

Inherited from AnyRef

Inherited from Any

Ungrouped