ScriptSubFeed

Instance Constructors

new ScriptSubFeed(parameters: Option[Map[String, String]] = None, dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues], isDAGStart: Boolean = false, isSkipped: Boolean = false)

parameters
arbitrary informations as key/value to pass on
dataObjectId
id of the DataObject this SubFeed corresponds to
partitionValues
Values of Partitions transported by this SubFeed
isDAGStart
true if this subfeed is a start node of the dag
isSkipped
true if this subfeed is the result of a skipped action

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def applyExecutionModeResultForInput(result: ExecutionModeResult, mainInputId: DataObjectId)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
def applyExecutionModeResultForOutput(result: ExecutionModeResult)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
final def asInstanceOf[T0]: T0

Definition Classes
Any
def breakLineage(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

Break lineage.
Break lineage. This means to discard an existing DataFrame or List of FileRefs, so that it is requested again from the DataObject. On one side this is usable to break long DataFrame Lineages over multiple Actions and instead reread the data from an intermediate table. On the other side it is needed if partition values or filter condition are changed.

Definition Classes
ScriptSubFeed → SubFeed
def clearDAGStart(): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
def clearPartitionValues(breakLineageOnChange: Boolean = true)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
def clearSkipped(): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val dataObjectId: DataObjectId

id of the DataObject this SubFeed corresponds to
id of the DataObject this SubFeed corresponds to

Definition Classes
ScriptSubFeed → SubFeed
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
val isDAGStart: Boolean

true if this subfeed is a start node of the dag
true if this subfeed is a start node of the dag

Definition Classes
ScriptSubFeed → SubFeed
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val isSkipped: Boolean

true if this subfeed is the result of a skipped action
true if this subfeed is the result of a skipped action

Definition Classes
ScriptSubFeed → SubFeed
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val parameters: Option[Map[String, String]]

arbitrary informations as key/value to pass on
val partitionValues: Seq[PartitionValues]

Values of Partitions transported by this SubFeed
Values of Partitions transported by this SubFeed

Definition Classes
ScriptSubFeed → SubFeed
def resultId: String

Definition Classes
SubFeed → DAGResult
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toOutput(dataObjectId: DataObjectId): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
def union(other: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): SubFeed

Definition Classes
ScriptSubFeed → SubFeed
def unionPartitionValues(otherPartitionValues: Seq[PartitionValues]): Seq[PartitionValues]

Definition Classes
SubFeed
def updatePartitionValues(partitions: Seq[String], breakLineageOnChange: Boolean = true, newPartitionValues: Option[Seq[PartitionValues]] = None)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

Definition Classes
ScriptSubFeed → SubFeed
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object ScriptSubFeed | package workflow

case class ScriptSubFeed(parameters: Option[Map[String, String]] = None, dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues], isDAGStart: Boolean = false, isSkipped: Boolean = false) extends SubFeed with Product with Serializable

Instance Constructors

new ScriptSubFeed(parameters: Option[Map[String, String]] = None, dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues], isDAGStart: Boolean = false, isSkipped: Boolean = false)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def applyExecutionModeResultForInput(result: ExecutionModeResult, mainInputId: DataObjectId)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

def applyExecutionModeResultForOutput(result: ExecutionModeResult)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

final def asInstanceOf[T0]: T0

def breakLineage(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

def clearDAGStart(): ScriptSubFeed

def clearPartitionValues(breakLineageOnChange: Boolean = true)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

def clearSkipped(): ScriptSubFeed

def clone(): AnyRef

val dataObjectId: DataObjectId

final def eq(arg0: AnyRef): Boolean

def finalize(): Unit

final def getClass(): Class[_]

val isDAGStart: Boolean

final def isInstanceOf[T0]: Boolean

val isSkipped: Boolean

lazy val logger: Logger

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val parameters: Option[Map[String, String]]

val partitionValues: Seq[PartitionValues]

def resultId: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toOutput(dataObjectId: DataObjectId): ScriptSubFeed

def union(other: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): SubFeed

def unionPartitionValues(otherPartitionValues: Seq[PartitionValues]): Seq[PartitionValues]

def updatePartitionValues(partitions: Seq[String], breakLineageOnChange: Boolean = true, newPartitionValues: Option[Seq[PartitionValues]] = None)(implicit session: SparkSession, context: ActionPipelineContext): ScriptSubFeed

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from SubFeed

Inherited from SmartDataLakeLogger

Inherited from DAGResult

Inherited from AnyRef

Inherited from Any

Ungrouped