final def !=(arg0: Any): Boolean

Definition Classes: AnyRef → Any

final def ##(): Int

Definition Classes: AnyRef → Any

final def ==(arg0: Any): Boolean

Definition Classes: AnyRef → Any

def addParents[In <: Serializable, Mid1 <: Serializable, Mid2 <: Serializable, Out <: Serializable](stage: Stage[In, Mid1], parent: Stage[Mid2, Out]): PipelineBuilder

Add a parent to a node.

In: Incoming type from 'start' stage.
Mid1: Intermediate type which is output for 'start' stage.
Mid2: Intermediate type which is output for 'end' stage.
Out: Outgoing type for 'end' stage.
stage: The child stage.
parent: The parent stage.
returns: The builder instance.

def addParents[In <: Serializable, Out <: Serializable](stage: Stage[In, Out], parents: StageList): PipelineBuilder

Add multiple parents (stages) in given ordered list.

In: Incoming type of the stage.
Out: Outgoing type of the stage.
stage: Stage to add the parents to.
parents: Parents to connect to the child stage.
returns: The builder instance.

def append[In <: Serializable](trans: (DataStream[In]) ⇒ Any)(implicit arg0: ClassTag[In], arg1: scala.reflect.api.JavaUniverse.TypeTag[In]): PipelineBuilder

Append a stage created from an output function.

In: Type of the DataStream.
trans: Output function.
returns: The builder instance.

def append[In <: Serializable, Out <: Serializable](trans: (DataStream[In]) ⇒ DataStream[Out])(implicit arg0: ClassTag[In], arg1: scala.reflect.api.JavaUniverse.TypeTag[In], arg2: ClassTag[Out], arg3: scala.reflect.api.JavaUniverse.TypeTag[Out]): PipelineBuilder

Append a stage created from a transform function.

In: Incoming type of the DataStream.
Out: Outgoing type of the DataStream.
trans: Transform function from one type to another.
returns: The builder instance.

def append[IN <: Serializable, OUT <: Serializable](stage: Stage[IN, OUT]): PipelineBuilder

Append a Stage in a sequential pipeline.

IN: Incoming type of the stage.
OUT: Outgoing type of the stage.
stage: The new stage to add.
returns: This builder instance.

def appendSource[In <: Serializable](input: (StreamExecutionEnvironment) ⇒ DataStream[In])(implicit arg0: ClassTag[In], arg1: scala.reflect.api.JavaUniverse.TypeTag[In]): PipelineBuilder

Append a stage created from an input function.

In: Type of the DataStream.
input: Input function.
returns: The builder instance.

final def asInstanceOf[T0]: T0

Definition Classes: Any

var bufferProperties: Properties

Properties of the buffer

Attributes: protected[org.codefeedr.pipeline]

var bufferType: BufferType

Type of buffer used in the pipeline

Attributes: protected

def build(): Pipeline

Builds a pipeline from the builder configuration.

returns: A new Pipeline.

Exceptions thrown: EmptyPipelineException When no pipeline is defined (graph is empty).

var checkpointing: Option[Long]

The Checkpointing interval.

The Checkpointing interval. Default: None (No checkpointing).

Attributes: protected

var checkpointingMode: CheckpointingMode

The checkpointing mode.

The checkpointing mode. Default is exactly once.

Attributes: protected

def clone(): AnyRef

Attributes: protected[java.lang]
Definition Classes: AnyRef
Annotations: @native() @throws( ... )

def disablePipelineVerification(): PipelineBuilder

Disables the pipeline verification.

Disables the pipeline verification. This is not recommended, it allows for nasty pipelines.

returns: The builder instance.

def edge(from: List[Stage[_ <: Serializable, _ <: Serializable]], to: List[Stage[_ <: Serializable, _ <: Serializable]]): Unit

Links multiple stages to multiple stages.

from: The list of stages to start from.
to: The list of stages to direct to.

def edge[In <: Serializable, Out <: Serializable](from: List[Stage[_ <: Serializable, _ <: Serializable]], to: Stage[In, Out]): PipelineBuilder

Directs multiple stages to one stage.

from: The list of stages to start from.
to: The stage to end.
returns: The builder instance.

def edge[In <: Serializable, Out <: Serializable](from: Stage[In, Out], to: List[Stage[_ <: Serializable, _ <: Serializable]]): PipelineBuilder

Directs a stage to multiple other stages.

from: The stage to start from.
to: The list of stages to direct to.
returns: The builder instance.

def edge[In <: Serializable, Mid1 <: Serializable, Mid2 <: Serializable, Out <: Serializable](from: Stage[In, Mid1], to: Stage[Mid2, Out]): PipelineBuilder

Creates an edge between two stages.

Creates an edge between two stages. The 'to' must not already have a parent.

If the graph is not configured yet (has no nodes), the graph is switched to a DAG automatically. If it was already configured as sequential, it will throw an IllegalStateException.

In: Incoming type from 'start' stage.
Mid1: Intermediate type which is output for 'start' stage.
Mid2: Intermediate type which is output for 'end' stage.
Out: Outgoing type for 'end' stage.
from: The stage to start from.
to: The stage to end at.

def enableCheckpointing(interval: Long): PipelineBuilder

Enable checkpointing for this pipeline.

interval: The interval to checkpoint on.
returns: This builder instance.

def enableCheckpointing(interval: Long, checkpointingMode: CheckpointingMode): PipelineBuilder

Enable checkpointing for this pipeline.

interval: The interval to checkpoint on.
checkpointingMode: The checkpointingmode (exactly once or at least once).
returns: This builder instance.

final def eq(arg0: AnyRef): Boolean

Definition Classes: AnyRef

def equals(arg0: Any): Boolean

Definition Classes: AnyRef → Any

def finalize(): Unit

Attributes: protected[java.lang]
Definition Classes: AnyRef
Annotations: @throws( classOf[java.lang.Throwable] )

def getBufferType: BufferType

Get the type of the buffer.

returns: The buffer type.

final def getClass(): Class[_]

Definition Classes: AnyRef → Any
Annotations: @native()

def getPipelineType: PipelineType

Get the type of the pipeline.

returns: Type of pipeline: Sequential or DAG.

var graph: DirectedAcyclicGraph

Graph of the pipeline

Attributes: protected[org.codefeedr.pipeline]

def hashCode(): Int

Definition Classes: AnyRef → Any
Annotations: @native()

final def isInstanceOf[T0]: Boolean

Definition Classes: Any

var keyManager: KeyManager

Key manager

Attributes: protected

val logger: Logger

Attributes: protected
Definition Classes: Logging

var name: String

The name of the pipeline, "CodeFeedr pipeline" by default.

Attributes: protected

final def ne(arg0: AnyRef): Boolean

Definition Classes: AnyRef

final def notify(): Unit

Definition Classes: AnyRef
Annotations: @native()

final def notifyAll(): Unit

Definition Classes: AnyRef
Annotations: @native()

var pipelineType: PipelineType

Type of the pipeline graph

Attributes: protected

var pipelineVerificationToggle: Boolean

Pipeline verification toggle, defaults to true *

Attributes: protected

var restartStrategy: RestartStrategyConfiguration

The RestartStrategy.

The RestartStrategy. Default: RestartStrategies.noRestart()

Attributes: protected

def setBufferProperty(key: String, value: String): PipelineBuilder

Set a buffer property.

Set a buffer property. A buffer property is generic for all buffers (like the serializer).

key: Key of the property.
value: Value of the property.
returns: This builder instance.

def setBufferType(bufferType: BufferType): PipelineBuilder

Set the type of the buffer.

bufferType: The new buffer type.
returns: The builder instance.

def setCheckpointingMode(checkpointingMode: CheckpointingMode): PipelineBuilder

Sets the CheckpointMode for this pipeline.

Sets the CheckpointMode for this pipeline. Note: this method does not enable checkpointing.

checkpointingMode: The checkpointingmode (exactly once or at least once).

def setKeyManager(km: KeyManager): PipelineBuilder

Set a KeyManager.

Set a KeyManager. A key manager handles API key management and can be accessed by every stage.

km: The key manager instance.
returns: This builder instance.

def setPipelineName(name: String): PipelineBuilder

Set name of the pipeline.

name: Name of the pipeline.
returns: This builder instance.

def setPipelineType(pipelineType: PipelineType): PipelineBuilder

Set the type of the pipeline.

pipelineType: Type of the pipeline.
returns: This builder instance.

def setRestartStrategy(strategy: RestartStrategyConfiguration): PipelineBuilder

Set the RestartStrategy of the whole pipeline.

strategy: The strategy.
returns: The builder instance.

def setSerializer(serializer: SerializerType): PipelineBuilder

Sets the serializer type for the buffer.

serializer: The serializer type (which is basically a string).
returns: This builder instance.

def setStageProperty(stage: Stage[_, _], key: String, value: String): PipelineBuilder

Set a stage property.

key: Key of the property.
value: Value of the property.
returns: This builder instance.

def setStateBackend(stateBackend: StateBackend): PipelineBuilder

Sets the StateBackend of the whole pipeline.

stateBackend: the statebackend.
returns: The builder instance.

def setStreamTimeCharacteristic(timeCharacteristic: TimeCharacteristic): PipelineBuilder

Set TimeCharacteristic of the whole pipeline.

timeCharacteristic: The TimeCharacterisic.
returns: This builder instance.

val stageProperties: HashMap[String, Properties]

Stage properties

Attributes: protected

var stateBackend: StateBackend

The StateBackend.

The StateBackend. Default: org.apache.flink.runtime.state.memory.MemoryStateBackend

Attributes: protected

var streamTimeCharacteristic: TimeCharacteristic

The StreamTimeCharacteristic.

The StreamTimeCharacteristic. Default: TimeCharacteristic.EventTime

Attributes: protected

final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes: AnyRef

def toString(): String

Definition Classes: AnyRef → Any

final def wait(): Unit

Definition Classes: AnyRef
Annotations: @throws( ... )

final def wait(arg0: Long, arg1: Int): Unit

Definition Classes: AnyRef
Annotations: @throws( ... )

final def wait(arg0: Long): Unit

Definition Classes: AnyRef
Annotations: @native() @throws( ... )

Packages

PipelineBuilder

class PipelineBuilder extends Logging

Instance Constructors

Value Members

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped